use of com.netflix.titus.supplementary.relocation.connector.TitusNode in project titus-control-plane by Netflix.
the class RelocationConnectorStubs method place.
public RelocationConnectorStubs place(String instanceGroupId, Task... tasks) {
List<TitusNode> nodes = new ArrayList<>(nodeDataResolver.getNodes(instanceGroupId).values());
int counter = 0;
for (Task task : tasks) {
TitusNode node = nodes.get(counter++ % nodes.size());
jobComponentStub.place(task.getId(), node.getId(), node.getIpAddress());
}
return this;
}
use of com.netflix.titus.supplementary.relocation.connector.TitusNode in project titus-control-plane by Netflix.
the class RelocationConnectorStubs method markNodeRelocationRequired.
public void markNodeRelocationRequired(String nodeId) {
TitusNode node = Preconditions.checkNotNull(nodeDataResolver.getNode(nodeId));
nodeDataResolver.addNode(node.toBuilder().withRelocationRequired(true).build());
}
use of com.netflix.titus.supplementary.relocation.connector.TitusNode in project titus-control-plane by Netflix.
the class DefaultDeschedulerServiceTest method verifyRelocationPlan.
private void verifyRelocationPlan(long relocationDelay, String reasonMessage) {
ReadOnlyJobOperations jobOperations = mock(ReadOnlyJobOperations.class);
DefaultDeschedulerService dds = new DefaultDeschedulerService(jobOperations, mock(ReadOnlyEvictionOperations.class), new KubernetesNodeDataResolver(configuration, TestDataFactory.mockFabric8IOConnector(), node -> true), () -> "foo|bar", titusRuntime);
Job<ServiceJobExt> job = JobGenerator.serviceJobs(oneTaskServiceJobDescriptor().but(ofServiceSize(2), withDisruptionBudget(budget(selfManagedPolicy(relocationDelay), unlimitedRate(), Collections.emptyList())))).getValue();
ServiceJobTask task = JobGenerator.serviceTasks(job).getValue();
when(jobOperations.getJob(job.getId())).thenReturn(Optional.of(job));
TitusNode node = TitusNode.newBuilder().withId("node1").withServerGroupId("asg1").withRelocationRequired(true).withBadCondition(false).build();
// Advance test clock
long clockAdvancedMs = 5_000;
TestClock testClock = (TestClock) titusRuntime.getClock();
testClock.advanceTime(Duration.ofMillis(clockAdvancedMs));
Optional<TaskRelocationPlan> relocationPlanForTask = dds.getRelocationPlanForTask(node, task, Collections.emptyMap());
assertThat(relocationPlanForTask).isPresent();
assertThat(relocationPlanForTask.get().getTaskId()).isEqualTo(task.getId());
// relocation time is expected to be decision clock time + retentionTimeMs
assertThat(relocationPlanForTask.get().getRelocationTime()).isEqualTo(relocationDelay + clockAdvancedMs);
assertThat(relocationPlanForTask.get().getDecisionTime()).isEqualTo(clockAdvancedMs);
assertThat(relocationPlanForTask.get().getReasonMessage()).isEqualTo(reasonMessage);
}
use of com.netflix.titus.supplementary.relocation.connector.TitusNode in project titus-control-plane by Netflix.
the class RelocationUtilTest method buildTasksFromNodesAndJobsFilter.
@Test
public void buildTasksFromNodesAndJobsFilter() {
String node1 = "node1";
String node2 = "node2";
String node3 = "node3";
Job<BatchJobExt> job1 = JobGenerator.oneBatchJob();
Job<BatchJobExt> job2 = JobGenerator.oneBatchJob();
Job<BatchJobExt> job3 = JobGenerator.oneBatchJob();
BatchJobTask task1 = JobGenerator.batchTasks(job1).getValue().toBuilder().addToTaskContext(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, node1).build();
BatchJobTask task2 = JobGenerator.batchTasks(job2).getValue().toBuilder().addToTaskContext(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, node2).build();
BatchJobTask task3 = JobGenerator.batchTasks(job3).getValue().toBuilder().addToTaskContext(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, node3).build();
ReadOnlyJobOperations jobOperations = mock(ReadOnlyJobOperations.class);
when(jobOperations.getJobs()).thenReturn(Arrays.asList(job1, job2, job3));
when(jobOperations.getTasks(job1.getId())).thenReturn(Collections.singletonList(task1));
when(jobOperations.getTasks(job2.getId())).thenReturn(Collections.singletonList(task2));
when(jobOperations.getTasks(job3.getId())).thenReturn(Collections.singletonList(task3));
Map<String, TitusNode> nodes = new HashMap<>(3);
nodes.put(node1, buildNode(node1));
nodes.put(node2, buildNode(node2));
nodes.put(node3, buildNode(node3));
Set<String> jobIds = new HashSet<>(2);
jobIds.addAll(Arrays.asList(job1.getId(), job3.getId()));
List<String> taskIdsOnBadNodes = RelocationUtil.buildTasksFromNodesAndJobsFilter(nodes, jobIds, jobOperations);
assertThat(taskIdsOnBadNodes.size()).isEqualTo(2);
}
use of com.netflix.titus.supplementary.relocation.connector.TitusNode in project titus-control-plane by Netflix.
the class DefaultNodeConditionControllerTest method checkTasksTerminatedDueToBadNodeConditions.
@Test
public void checkTasksTerminatedDueToBadNodeConditions() {
// Mock jobs, tasks & nodes
Map<String, TitusNode> nodeMap = buildNodes();
List<Job<BatchJobExt>> jobs = getJobs(true);
Map<String, List<Task>> tasksByJobIdMap = buildTasksForJobAndNodeAssignment(new ArrayList<>(nodeMap.values()), jobs);
TitusRuntime titusRuntime = mock(TitusRuntime.class);
when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
RelocationConfiguration configuration = mock(RelocationConfiguration.class);
when(configuration.getBadNodeConditionPattern()).thenReturn(".*Failure");
when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
when(nodeDataResolver.resolve()).thenReturn(nodeMap);
JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
when(jobDataReplicator.getStalenessMs()).thenReturn(0L);
ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
when(readOnlyJobOperations.getJobs()).thenReturn(new ArrayList<>(jobs));
tasksByJobIdMap.forEach((key, value) -> when(readOnlyJobOperations.getTasks(key)).thenReturn(value));
JobManagementClient jobManagementClient = mock(JobManagementClient.class);
Set<String> terminatedTaskIds = new HashSet<>();
when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
String taskIdToBeTerminated = invocation.getArgument(0);
terminatedTaskIds.add(taskIdToBeTerminated);
return Mono.empty();
});
DefaultNodeConditionController nodeConditionCtrl = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
StepVerifier.create(nodeConditionCtrl.handleNodesWithBadCondition(executionContext)).verifyComplete();
assertThat(terminatedTaskIds).isNotEmpty();
assertThat(terminatedTaskIds.size()).isEqualTo(2);
verifyTerminatedTasksOnBadNodes(terminatedTaskIds, tasksByJobIdMap, nodeMap);
}
Aggregations