Examples with EntityHolder com.netflix.titus.common.framework.reconciler.EntityHolder used on opensource projects

Example 6 with EntityHolder

use of com.netflix.titus.common.framework.reconciler.EntityHolder in project titus-control-plane by Netflix.

the class JobReconciliationFrameworkFactory method newRestoredEngine.

private InternalReconciliationEngine<JobManagerReconcilerEvent> newRestoredEngine(Job job, List<Task> tasks) {
    EntityHolder jobHolder = EntityHolder.newRoot(job.getId(), job);
    for (Task task : tasks) {
        EntityHolder taskHolder = EntityHolder.newRoot(task.getId(), task);
        EntityHolder decorated = TaskTimeoutChangeActions.setTimeoutOnRestoreFromStore(jobManagerConfiguration, taskHolder, clock);
        jobHolder = jobHolder.addChild(decorated);
    }
    return newEngine(jobHolder, false);
}

Example 7 with EntityHolder

use of com.netflix.titus.common.framework.reconciler.EntityHolder in project titus-control-plane by Netflix.

the class DefaultV3JobOperations method moveServiceTask.

@Override
public Observable<Void> moveServiceTask(String sourceJobId, String targetJobId, String taskId, CallMetadata callMetadata) {
    return Observable.defer(() -> {
        Pair<ReconciliationEngine<JobManagerReconcilerEvent>, EntityHolder> fromEngineTaskPair = reconciliationFramework.findEngineByChildId(taskId).orElseThrow(() -> JobManagerException.taskNotFound(taskId));
        ReconciliationEngine<JobManagerReconcilerEvent> engineFrom = fromEngineTaskPair.getLeft();
        Job<ServiceJobExt> jobFrom = engineFrom.getReferenceView().getEntity();
        if (!JobFunctions.isServiceJob(jobFrom)) {
            throw JobManagerException.notServiceJob(jobFrom.getId());
        }
        if (!jobFrom.getId().equals(sourceJobId)) {
            throw JobManagerException.taskJobMismatch(taskId, sourceJobId);
        }
        if (jobFrom.getId().equals(targetJobId)) {
            throw JobManagerException.sameJobs(jobFrom.getId());
        }
        ReconciliationEngine<JobManagerReconcilerEvent> engineTo = reconciliationFramework.findEngineByRootId(targetJobId).orElseThrow(() -> JobManagerException.jobNotFound(targetJobId));
        Job<ServiceJobExt> jobTo = engineTo.getReferenceView().getEntity();
        if (!JobFunctions.isServiceJob(jobTo)) {
            throw JobManagerException.notServiceJob(jobTo.getId());
        }
        JobCompatibility compatibility = JobCompatibility.of(jobFrom, jobTo);
        if (featureActivationConfiguration.isMoveTaskValidationEnabled() && !compatibility.isCompatible()) {
            Optional<String> diffReport = ProtobufExt.diffReport(GrpcJobManagementModelConverters.toGrpcJobDescriptor(compatibility.getNormalizedDescriptorFrom()), GrpcJobManagementModelConverters.toGrpcJobDescriptor(compatibility.getNormalizedDescriptorTo()));
            throw JobManagerException.notCompatible(jobFrom, jobTo, diffReport.orElse(""));
        }
        return reconciliationFramework.changeReferenceModel(new MoveTaskBetweenJobsAction(engineFrom, engineTo, taskId, store, callMetadata, versionSupplier), (rootId, modelUpdatesObservable) -> {
            String name;
            String summary;
            if (targetJobId.equals(rootId)) {
                name = "moveTask(to)";
                summary = "Moving a task to this job from job " + jobFrom.getId();
            } else {
                name = "moveTask(from)";
                summary = "Moving a task out of this job to job " + jobTo.getId();
            }
            return new TitusChangeAction(Trigger.API, rootId, null, name, summary, callMetadata) {

                @Override
                public Observable<List<ModelActionHolder>> apply() {
                    return modelUpdatesObservable;
                }
            };
        }, jobFrom.getId(), jobTo.getId());
    });
}

Example 8 with EntityHolder

use of com.netflix.titus.common.framework.reconciler.EntityHolder in project titus-control-plane by Netflix.

the class MoveTaskBetweenJobsAction method apply.

@Override
public Observable<Map<String, List<ModelActionHolder>>> apply() {
    return Observable.defer(() -> {
        // Validate data
        Job<ServiceJobExt> jobFrom = engineFrom.getReferenceView().getEntity();
        Job<ServiceJobExt> jobTo = engineTo.getReferenceView().getEntity();
        EntityHolder taskFromReferenceHolder = engineFrom.getReferenceView().findChildById(taskId).orElseThrow(() -> JobManagerException.taskJobMismatch(taskId, jobFrom.getId()));
        if (jobFrom.getStatus().getState() != JobState.Accepted) {
            throw JobManagerException.unexpectedJobState(jobTo, JobState.Accepted);
        }
        Capacity capacityFrom = jobFrom.getJobDescriptor().getExtensions().getCapacity();
        if (capacityFrom.getMin() >= capacityFrom.getDesired()) {
            throw JobManagerException.belowMinCapacity(jobFrom, 1);
        }
        if (jobTo.getStatus().getState() != JobState.Accepted) {
            throw JobManagerException.unexpectedJobState(jobTo, JobState.Accepted);
        }
        Capacity capacityTo = jobTo.getJobDescriptor().getExtensions().getCapacity();
        if (capacityTo.getDesired() >= capacityTo.getMax()) {
            throw JobManagerException.aboveMaxCapacity(jobTo, 1);
        }
        Task taskFromReference = taskFromReferenceHolder.getEntity();
        Optional<EntityHolder> taskFromRunningHolder = engineFrom.getRunningView().findChildById(taskId);
        // Compute new model entities
        // Decrement job size by 1
        Job<ServiceJobExt> updatedJobFrom = nextVersion(JobFunctions.incrementJobSize(jobFrom, -1), versionSupplier);
        Job<ServiceJobExt> updatedJobTo = nextVersion(JobFunctions.incrementJobSize(jobTo, 1), versionSupplier);
        Task updatedReferenceTaskTo = VersionSuppliers.nextVersion(JobFunctions.moveTask(jobFrom.getId(), jobTo.getId(), taskFromReference), versionSupplier);
        // Move the task
        return titusStore.moveTask(updatedJobFrom, updatedJobTo, updatedReferenceTaskTo).andThen(Observable.fromCallable(() -> ImmutableMap.of(jobFrom.getId(), createModelUpdateActionsFrom(updatedJobFrom, updatedJobTo, taskFromReference, callMetadata), jobTo.getId(), createModelUpdateActionsTo(updatedJobFrom, updatedJobTo, updatedReferenceTaskTo, taskFromRunningHolder, callMetadata))));
    });
}

Example 9 with EntityHolder

use of com.netflix.titus.common.framework.reconciler.EntityHolder in project titus-control-plane by Netflix.

the class BatchDifferenceResolver method createNewTaskAction.

private Optional<TitusChangeAction> createNewTaskAction(BatchJobView refJobView, int taskIndex, Optional<EntityHolder> previousTask, List<String> unassignedIpAllocations, List<String> ebsVolumeIds) {
    // Safety check
    long numberOfNotFinishedTasks = refJobView.getJobHolder().getChildren().stream().filter(holder -> TaskState.isRunning(((Task) holder.getEntity()).getStatus().getState())).count();
    if (numberOfNotFinishedTasks >= refJobView.getRequiredSize()) {
        titusRuntime.getCodeInvariants().inconsistent("Batch job reconciler attempts to create too many tasks: jobId=%s, requiredSize=%s, current=%s", refJobView.getJob().getId(), refJobView.getRequiredSize(), numberOfNotFinishedTasks);
        return Optional.empty();
    }
    Map<String, String> taskContext = getTaskContext(previousTask, unassignedIpAllocations, ebsVolumeIds);
    JobDescriptor jobDescriptor = refJobView.getJob().getJobDescriptor();
    ApplicationSLA capacityGroupDescriptor = JobManagerUtil.getCapacityGroupDescriptor(jobDescriptor, capacityGroupService);
    String resourcePool = capacityGroupDescriptor.getResourcePool();
    taskContext = CollectionsExt.copyAndAdd(taskContext, ImmutableMap.of(TaskAttributes.TASK_ATTRIBUTES_RESOURCE_POOL, resourcePool, TaskAttributes.TASK_ATTRIBUTES_TIER, capacityGroupDescriptor.getTier().name()));
    TitusChangeAction storeAction = storeWriteRetryInterceptor.apply(createOrReplaceTaskAction(runtime, jobStore, refJobView.getJobHolder(), taskIndex, versionSupplier, clock, taskContext));
    return Optional.of(storeAction);
}

Example 10 with EntityHolder

use of com.netflix.titus.common.framework.reconciler.EntityHolder in project titus-control-plane by Netflix.

the class BatchDifferenceResolver method applyRuntime.

private List<ChangeAction> applyRuntime(ReconciliationEngine<JobManagerReconcilerEvent> engine, BatchJobView refJobView, EntityHolder runningModel, EntityHolder storeModel, AtomicInteger allowedNewTasks) {
    List<ChangeAction> actions = new ArrayList<>();
    EntityHolder referenceModel = refJobView.getJobHolder();
    BatchJobView runningJobView = new BatchJobView(runningModel);
    if (DifferenceResolverUtils.hasJobState(referenceModel, JobState.KillInitiated)) {
        List<ChangeAction> killInitiatedActions = KillInitiatedActions.reconcilerInitiatedAllTasksKillInitiated(engine, runtime, jobStore, TaskStatus.REASON_TASK_KILLED, "Killing task as its job is in KillInitiated state", configuration.getConcurrentReconcilerStoreUpdateLimit(), versionSupplier, titusRuntime);
        if (killInitiatedActions.isEmpty()) {
            return DifferenceResolverUtils.findTaskStateTimeouts(engine, runningJobView, configuration, runtime, jobStore, versionSupplier, stuckInStateRateLimiter, titusRuntime);
        }
        return killInitiatedActions;
    } else if (DifferenceResolverUtils.hasJobState(referenceModel, JobState.Finished)) {
        return Collections.emptyList();
    }
    List<ChangeAction> numberOfTaskAdjustingActions = findJobSizeInconsistencies(refJobView, storeModel, allowedNewTasks);
    actions.addAll(numberOfTaskAdjustingActions);
    if (numberOfTaskAdjustingActions.isEmpty()) {
        actions.addAll(findMissingRunningTasks(engine, refJobView, runningJobView));
    }
    actions.addAll(DifferenceResolverUtils.findTaskStateTimeouts(engine, runningJobView, configuration, runtime, jobStore, versionSupplier, stuckInStateRateLimiter, titusRuntime));
    return actions;
}