Examples with ReadOnlyJobOperations com.netflix.titus.api.jobmanager.service.ReadOnlyJobOperations used on opensource projects

Example 1 with ReadOnlyJobOperations

use of com.netflix.titus.api.jobmanager.service.ReadOnlyJobOperations in project titus-control-plane by Netflix.

the class DefaultDeschedulerServiceTest method verifyRelocationPlan.

private void verifyRelocationPlan(long relocationDelay, String reasonMessage) {
    ReadOnlyJobOperations jobOperations = mock(ReadOnlyJobOperations.class);
    DefaultDeschedulerService dds = new DefaultDeschedulerService(jobOperations, mock(ReadOnlyEvictionOperations.class), new KubernetesNodeDataResolver(configuration, TestDataFactory.mockFabric8IOConnector(), node -> true), () -> "foo|bar", titusRuntime);
    Job<ServiceJobExt> job = JobGenerator.serviceJobs(oneTaskServiceJobDescriptor().but(ofServiceSize(2), withDisruptionBudget(budget(selfManagedPolicy(relocationDelay), unlimitedRate(), Collections.emptyList())))).getValue();
    ServiceJobTask task = JobGenerator.serviceTasks(job).getValue();
    when(jobOperations.getJob(job.getId())).thenReturn(Optional.of(job));
    TitusNode node = TitusNode.newBuilder().withId("node1").withServerGroupId("asg1").withRelocationRequired(true).withBadCondition(false).build();
    // Advance test clock
    long clockAdvancedMs = 5_000;
    TestClock testClock = (TestClock) titusRuntime.getClock();
    testClock.advanceTime(Duration.ofMillis(clockAdvancedMs));
    Optional<TaskRelocationPlan> relocationPlanForTask = dds.getRelocationPlanForTask(node, task, Collections.emptyMap());
    assertThat(relocationPlanForTask).isPresent();
    assertThat(relocationPlanForTask.get().getTaskId()).isEqualTo(task.getId());
    // relocation time is expected to be decision clock time + retentionTimeMs
    assertThat(relocationPlanForTask.get().getRelocationTime()).isEqualTo(relocationDelay + clockAdvancedMs);
    assertThat(relocationPlanForTask.get().getDecisionTime()).isEqualTo(clockAdvancedMs);
    assertThat(relocationPlanForTask.get().getReasonMessage()).isEqualTo(reasonMessage);
}

Example 2 with ReadOnlyJobOperations

use of com.netflix.titus.api.jobmanager.service.ReadOnlyJobOperations in project titus-control-plane by Netflix.

the class RelocationUtilTest method buildTasksFromNodesAndJobsFilter.

@Test
public void buildTasksFromNodesAndJobsFilter() {
    String node1 = "node1";
    String node2 = "node2";
    String node3 = "node3";
    Job<BatchJobExt> job1 = JobGenerator.oneBatchJob();
    Job<BatchJobExt> job2 = JobGenerator.oneBatchJob();
    Job<BatchJobExt> job3 = JobGenerator.oneBatchJob();
    BatchJobTask task1 = JobGenerator.batchTasks(job1).getValue().toBuilder().addToTaskContext(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, node1).build();
    BatchJobTask task2 = JobGenerator.batchTasks(job2).getValue().toBuilder().addToTaskContext(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, node2).build();
    BatchJobTask task3 = JobGenerator.batchTasks(job3).getValue().toBuilder().addToTaskContext(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, node3).build();
    ReadOnlyJobOperations jobOperations = mock(ReadOnlyJobOperations.class);
    when(jobOperations.getJobs()).thenReturn(Arrays.asList(job1, job2, job3));
    when(jobOperations.getTasks(job1.getId())).thenReturn(Collections.singletonList(task1));
    when(jobOperations.getTasks(job2.getId())).thenReturn(Collections.singletonList(task2));
    when(jobOperations.getTasks(job3.getId())).thenReturn(Collections.singletonList(task3));
    Map<String, TitusNode> nodes = new HashMap<>(3);
    nodes.put(node1, buildNode(node1));
    nodes.put(node2, buildNode(node2));
    nodes.put(node3, buildNode(node3));
    Set<String> jobIds = new HashSet<>(2);
    jobIds.addAll(Arrays.asList(job1.getId(), job3.getId()));
    List<String> taskIdsOnBadNodes = RelocationUtil.buildTasksFromNodesAndJobsFilter(nodes, jobIds, jobOperations);
    assertThat(taskIdsOnBadNodes.size()).isEqualTo(2);
}

Example 3 with ReadOnlyJobOperations

use of com.netflix.titus.api.jobmanager.service.ReadOnlyJobOperations in project titus-control-plane by Netflix.

the class DefaultNodeConditionControllerTest method checkTasksTerminatedDueToBadNodeConditions.

@Test
public void checkTasksTerminatedDueToBadNodeConditions() {
    // Mock jobs, tasks & nodes
    Map<String, TitusNode> nodeMap = buildNodes();
    List<Job<BatchJobExt>> jobs = getJobs(true);
    Map<String, List<Task>> tasksByJobIdMap = buildTasksForJobAndNodeAssignment(new ArrayList<>(nodeMap.values()), jobs);
    TitusRuntime titusRuntime = mock(TitusRuntime.class);
    when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
    RelocationConfiguration configuration = mock(RelocationConfiguration.class);
    when(configuration.getBadNodeConditionPattern()).thenReturn(".*Failure");
    when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
    NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
    when(nodeDataResolver.resolve()).thenReturn(nodeMap);
    JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
    when(jobDataReplicator.getStalenessMs()).thenReturn(0L);
    ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
    when(readOnlyJobOperations.getJobs()).thenReturn(new ArrayList<>(jobs));
    tasksByJobIdMap.forEach((key, value) -> when(readOnlyJobOperations.getTasks(key)).thenReturn(value));
    JobManagementClient jobManagementClient = mock(JobManagementClient.class);
    Set<String> terminatedTaskIds = new HashSet<>();
    when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
        String taskIdToBeTerminated = invocation.getArgument(0);
        terminatedTaskIds.add(taskIdToBeTerminated);
        return Mono.empty();
    });
    DefaultNodeConditionController nodeConditionCtrl = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
    ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
    StepVerifier.create(nodeConditionCtrl.handleNodesWithBadCondition(executionContext)).verifyComplete();
    assertThat(terminatedTaskIds).isNotEmpty();
    assertThat(terminatedTaskIds.size()).isEqualTo(2);
    verifyTerminatedTasksOnBadNodes(terminatedTaskIds, tasksByJobIdMap, nodeMap);
}

Example 4 with ReadOnlyJobOperations

use of com.netflix.titus.api.jobmanager.service.ReadOnlyJobOperations in project titus-control-plane by Netflix.

the class DefaultNodeConditionControllerTest method noTerminationsOnDataStaleness.

@Test
public void noTerminationsOnDataStaleness() {
    TitusRuntime titusRuntime = mock(TitusRuntime.class);
    when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
    RelocationConfiguration configuration = mock(RelocationConfiguration.class);
    when(configuration.getBadNodeConditionPattern()).thenReturn(".*Problem");
    when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
    when(configuration.getDataStalenessThresholdMs()).thenReturn(8000L);
    NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
    when(nodeDataResolver.getStalenessMs()).thenReturn(5L);
    JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
    when(jobDataReplicator.getStalenessMs()).thenReturn(10L);
    ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
    JobManagementClient jobManagementClient = mock(JobManagementClient.class);
    Set<String> terminatedTaskIds = new HashSet<>();
    when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
        String taskIdToBeTerminated = invocation.getArgument(0);
        terminatedTaskIds.add(taskIdToBeTerminated);
        return Mono.empty();
    });
    DefaultNodeConditionController nodeConditionCtrl = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
    ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
    StepVerifier.create(nodeConditionCtrl.handleNodesWithBadCondition(executionContext)).verifyComplete();
    // No tasks terminated
    assertThat(terminatedTaskIds).isEmpty();
}

Example 5 with ReadOnlyJobOperations

use of com.netflix.titus.api.jobmanager.service.ReadOnlyJobOperations in project titus-control-plane by Netflix.

the class DefaultNodeConditionControllerTest method badNodeConditionsIgnoredForJobsNotOptingIn.

@Test
public void badNodeConditionsIgnoredForJobsNotOptingIn() {
    Map<String, TitusNode> nodeMap = buildNodes();
    List<Job<BatchJobExt>> jobs = getJobs(false);
    Map<String, List<Task>> stringListMap = buildTasksForJobAndNodeAssignment(new ArrayList<>(nodeMap.values()), jobs);
    TitusRuntime titusRuntime = mock(TitusRuntime.class);
    when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
    RelocationConfiguration configuration = mock(RelocationConfiguration.class);
    when(configuration.getBadNodeConditionPattern()).thenReturn(".*Failure");
    when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
    NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
    when(nodeDataResolver.resolve()).thenReturn(nodeMap);
    JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
    when(jobDataReplicator.getStalenessMs()).thenReturn(0L);
    // Job attribute "terminateContainerOnBadAgent" = False
    ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
    when(readOnlyJobOperations.getJobs()).thenReturn(new ArrayList<>(jobs));
    stringListMap.forEach((key, value) -> when(readOnlyJobOperations.getTasks(key)).thenReturn(value));
    JobManagementClient jobManagementClient = mock(JobManagementClient.class);
    Set<String> terminatedTaskIds = new HashSet<>();
    when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
        String taskIdToBeTerminated = invocation.getArgument(0);
        terminatedTaskIds.add(taskIdToBeTerminated);
        return Mono.empty();
    });
    DefaultNodeConditionController nodeConditionController = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
    ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
    StepVerifier.create(nodeConditionController.handleNodesWithBadCondition(executionContext)).verifyComplete();
    // no tasks should be terminated for jobs
    assertThat(terminatedTaskIds).isEmpty();
}