use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.
the class BatchDifferenceResolver method createNewTaskAction.
private Optional<TitusChangeAction> createNewTaskAction(BatchJobView refJobView, int taskIndex, Optional<EntityHolder> previousTask, List<String> unassignedIpAllocations, List<String> ebsVolumeIds) {
// Safety check
long numberOfNotFinishedTasks = refJobView.getJobHolder().getChildren().stream().filter(holder -> TaskState.isRunning(((Task) holder.getEntity()).getStatus().getState())).count();
if (numberOfNotFinishedTasks >= refJobView.getRequiredSize()) {
titusRuntime.getCodeInvariants().inconsistent("Batch job reconciler attempts to create too many tasks: jobId=%s, requiredSize=%s, current=%s", refJobView.getJob().getId(), refJobView.getRequiredSize(), numberOfNotFinishedTasks);
return Optional.empty();
}
Map<String, String> taskContext = getTaskContext(previousTask, unassignedIpAllocations, ebsVolumeIds);
JobDescriptor jobDescriptor = refJobView.getJob().getJobDescriptor();
ApplicationSLA capacityGroupDescriptor = JobManagerUtil.getCapacityGroupDescriptor(jobDescriptor, capacityGroupService);
String resourcePool = capacityGroupDescriptor.getResourcePool();
taskContext = CollectionsExt.copyAndAdd(taskContext, ImmutableMap.of(TaskAttributes.TASK_ATTRIBUTES_RESOURCE_POOL, resourcePool, TaskAttributes.TASK_ATTRIBUTES_TIER, capacityGroupDescriptor.getTier().name()));
TitusChangeAction storeAction = storeWriteRetryInterceptor.apply(createOrReplaceTaskAction(runtime, jobStore, refJobView.getJobHolder(), taskIndex, versionSupplier, clock, taskContext));
return Optional.of(storeAction);
}
use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.
the class ServiceDifferenceResolver method createNewTaskAction.
private Optional<TitusChangeAction> createNewTaskAction(ServiceJobView refJobView, Optional<EntityHolder> previousTask, List<String> unassignedIpAllocations, List<String> unassignedEbsVolumeIds) {
// Safety check
long numberOfNotFinishedTasks = getNumberOfNotFinishedTasks(refJobView);
if (numberOfNotFinishedTasks >= refJobView.getRequiredSize()) {
titusRuntime.getCodeInvariants().inconsistent("Service job reconciler attempts to create too many tasks: jobId=%s, requiredSize=%s, current=%s", refJobView.getJob().getId(), refJobView.getRequiredSize(), numberOfNotFinishedTasks);
return Optional.empty();
}
Map<String, String> taskContext = getTaskContext(previousTask, unassignedIpAllocations, unassignedEbsVolumeIds);
JobDescriptor jobDescriptor = refJobView.getJob().getJobDescriptor();
ApplicationSLA capacityGroupDescriptor = JobManagerUtil.getCapacityGroupDescriptor(jobDescriptor, capacityGroupService);
String resourcePool = capacityGroupDescriptor.getResourcePool();
taskContext = CollectionsExt.copyAndAdd(taskContext, ImmutableMap.of(TaskAttributes.TASK_ATTRIBUTES_RESOURCE_POOL, resourcePool, TaskAttributes.TASK_ATTRIBUTES_TIER, capacityGroupDescriptor.getTier().name()));
TitusChangeAction storeAction = storeWriteRetryInterceptor.apply(createOrReplaceTaskAction(runtime, jobStore, versionSupplier, refJobView.getJobHolder(), previousTask, clock, taskContext));
return Optional.of(storeAction);
}
use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.
the class JobIpAllocationsTest method testWaitingTaskContext.
/**
* Tests a job waiting for an in use IP allocation has updated task context fields.
*/
@Test(timeout = 30_000)
// TODO Read static IP allocation status from pod message field, and add it to the task context.
@Ignore
public void testWaitingTaskContext() throws Exception {
JobDescriptor<ServiceJobExt> firstIpJobDescriptor = ONE_TASK_SERVICE_JOB;
JobDescriptor<ServiceJobExt> secondIpJobDescriptor = firstIpJobDescriptor.but(j -> j.getJobGroupInfo().toBuilder().withSequence("v001"));
// Schedule the first task and ensure it's in the correct zone with the correct task context
jobsScenarioBuilder.schedule(firstIpJobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.startTasksInNewJob()).allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectTaskContext(TaskAttributes.TASK_ATTRIBUTES_IP_ALLOCATION_ID, getIpAllocationIdFromJob(0, firstIpJobDescriptor))).allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectZoneId(getZoneFromJobIpAllocation(0, firstIpJobDescriptor))));
String firstJobId = jobsScenarioBuilder.takeJob(0).getJobId();
// Schedule the second task and ensure it's blocked on the first task
jobsScenarioBuilder.schedule(secondIpJobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.jobAccepted()).expectAllTasksCreated().allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectStateUpdates(TaskStatus.TaskState.Accepted)));
String secondJobId = jobsScenarioBuilder.takeJob(1).getJobId();
// Query the gRPC endpoint and ensure the first task does not have a waiting task context field.
TaskQueryResult firstTaskQueryResult = client.findTasks(TaskQuery.newBuilder().setPage(Page.newBuilder().setPageSize(100).build()).putFilteringCriteria("jobIds", firstJobId).build());
assertThat(firstTaskQueryResult.getItemsCount()).isEqualTo(1);
firstTaskQueryResult.getItemsList().forEach(task -> {
assertThat(task.getTaskContextMap()).doesNotContainKeys(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION);
});
String firstTaskId = firstTaskQueryResult.getItems(0).getId();
// Query the gRPC endpoint and ensure the second task has a waiting task context field.
TaskQueryResult secondTaskQueryResult = client.findTasks(TaskQuery.newBuilder().setPage(Page.newBuilder().setPageSize(100).build()).putFilteringCriteria("jobIds", secondJobId).build());
assertThat(secondTaskQueryResult.getItemsCount()).isEqualTo(1);
secondTaskQueryResult.getItemsList().forEach(task -> {
assertThat(task.getTaskContextMap()).contains(new AbstractMap.SimpleImmutableEntry<>(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION, firstTaskId));
});
// Observe the second job and ensure the streamed task has a waiting task context field.
boolean verified = false;
Iterator<JobChangeNotification> it = client.observeJob(JobId.newBuilder().setId(secondJobId).build());
while (it.hasNext()) {
JobChangeNotification jobChangeNotification = it.next();
if (jobChangeNotification.hasTaskUpdate()) {
Map<String, String> taskContext = jobChangeNotification.getTaskUpdate().getTask().getTaskContextMap();
assertThat(taskContext).contains(new AbstractMap.SimpleImmutableEntry<>(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION, firstTaskId));
verified = true;
} else if (jobChangeNotification.hasSnapshotEnd()) {
break;
}
}
assertThat(verified).isTrue();
}
use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.
the class JobCriteriaQueryTest method testSearchByJobState.
@Test(timeout = 30_000)
public void testSearchByJobState() throws Exception {
JobDescriptor<BatchJobExt> jobDescriptor = batchJobDescriptors().getValue().toBuilder().withApplicationName("testSearchByJobState").build();
String acceptedJobId = jobsScenarioBuilder.scheduleAndReturnJob(jobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob())).getId();
String killInitiatedJobId = jobsScenarioBuilder.scheduleAndReturnJob(jobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob()).killJob().expectJobUpdateEvent(job -> job.getStatus().getState() == JobState.KillInitiated, "Expected state: " + JobState.KillInitiated)).getId();
String acceptedTaskId = jobsScenarioBuilder.takeJob(acceptedJobId).getTaskByIndex(0).getTask().getId();
String killInitiatedTaskId = jobsScenarioBuilder.takeJob(killInitiatedJobId).getTaskByIndex(0).getTask().getId();
// Indexes are recomputed after events are sent, so if we run findJobs/findTasks immediately, they may use stale index.
Thread.sleep(10);
JobQuery.Builder jobQueryBuilder = JobQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByJobState").setPage(PAGE);
TaskQuery.Builder taskQueryBuilder = TaskQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByJobState").setPage(PAGE);
// Jobs (Accepted)
JobQueryResult acceptedJobQueryResult = client.findJobs(jobQueryBuilder.putFilteringCriteria("jobState", "Accepted").build());
assertThat(acceptedJobQueryResult.getItemsList()).hasSize(1);
Job acceptedJobQueryResultItem = acceptedJobQueryResult.getItems(0);
assertThat(acceptedJobQueryResultItem.getId()).isEqualTo(acceptedJobId);
// Jobs (KillInitiated)
JobQueryResult killInitJobQueryResult = client.findJobs(jobQueryBuilder.putFilteringCriteria("jobState", "KillInitiated").setPage(PAGE).build());
assertThat(killInitJobQueryResult.getItemsList()).hasSize(1);
Job killInitJobQueryResultItem = killInitJobQueryResult.getItems(0);
assertThat(killInitJobQueryResultItem.getId()).isEqualTo(killInitiatedJobId);
// Tasks (Accepted)
TaskQueryResult acceptedTaskQueryResult = client.findTasks(taskQueryBuilder.putFilteringCriteria("jobState", "Accepted").setPage(PAGE).build());
assertThat(acceptedTaskQueryResult.getItemsList()).hasSize(1);
assertThat(acceptedTaskQueryResult.getItems(0).getId()).isEqualTo(acceptedTaskId);
// Tasks (KillInitiated)
TaskQueryResult killInitTaskQueryResult = client.findTasks(taskQueryBuilder.putFilteringCriteria("jobState", "KillInitiated").setPage(PAGE).build());
assertThat(killInitTaskQueryResult.getItemsList()).hasSize(1);
assertThat(killInitTaskQueryResult.getItems(0).getId()).isEqualTo(killInitiatedTaskId);
}
use of com.netflix.titus.api.jobmanager.model.job.JobDescriptor in project titus-control-plane by Netflix.
the class DefaultAppScaleManagerTest method mockV3Operations.
private V3JobOperations mockV3Operations(String jobIdOne, String jobIdTwo) {
V3JobOperations v3JobOperations = mock(V3JobOperations.class);
// FIXME Use JobGenerator instead of mocking.
Job jobOne = mock(Job.class);
when(jobOne.getId()).thenReturn(jobIdOne);
JobDescriptor jobDescriptorOne = mock(JobDescriptor.class);
ServiceJobExt serviceJobExtOne = mock(ServiceJobExt.class);
JobGroupInfo jobGroupInfoOne = buildMockJobGroupInfo(jobIdOne);
Capacity capacityOne = mock(Capacity.class);
when(capacityOne.getMax()).thenReturn(10);
when(capacityOne.getMin()).thenReturn(1);
when(serviceJobExtOne.getCapacity()).thenReturn(capacityOne);
when(jobDescriptorOne.getExtensions()).thenReturn(serviceJobExtOne);
when(jobOne.getJobDescriptor()).thenReturn(jobDescriptorOne);
when(jobDescriptorOne.getJobGroupInfo()).thenReturn(jobGroupInfoOne);
when(jobDescriptorOne.getApplicationName()).thenReturn("testApp1");
Job jobTwo = mock(Job.class);
when(jobTwo.getId()).thenReturn(jobIdTwo);
JobDescriptor jobDescriptorTwo = mock(JobDescriptor.class);
ServiceJobExt serviceJobExtTwo = mock(ServiceJobExt.class);
Capacity capacityJobTwo = mock(Capacity.class);
when(capacityJobTwo.getMin()).thenAnswer(new Answer<Integer>() {
private int count = 0;
@Override
public Integer answer(InvocationOnMock invocation) throws Throwable {
if (count++ < 2) {
return 1;
} else {
return 5;
}
}
});
when(capacityJobTwo.getMax()).thenAnswer(new Answer<Integer>() {
private int count = 0;
@Override
public Integer answer(InvocationOnMock invocation) throws Throwable {
if (count++ < 2) {
return 10;
} else {
return 15;
}
}
});
when(serviceJobExtTwo.getCapacity()).thenReturn(capacityJobTwo);
when(jobDescriptorTwo.getExtensions()).thenReturn(serviceJobExtTwo);
when(jobDescriptorTwo.getJobGroupInfo()).thenReturn(jobGroupInfoOne);
when(jobDescriptorTwo.getApplicationName()).thenReturn("testApp2");
when(jobTwo.getJobDescriptor()).thenReturn(jobDescriptorTwo);
when(jobTwo.getStatus()).thenReturn(JobModel.newJobStatus().withState(JobState.Accepted).build());
when(v3JobOperations.getJob(jobIdOne)).thenReturn(Optional.of(jobOne));
when(v3JobOperations.getJob(jobIdTwo)).thenReturn(Optional.of(jobTwo));
JobManagerEvent<?> jobUpdateEvent = JobUpdateEvent.newJob(jobTwo, callMetadata);
when(v3JobOperations.observeJobs()).thenAnswer(invocation -> Observable.from(asList(jobUpdateEvent)));
return v3JobOperations;
}
Aggregations