Examples with RelocationConfiguration com.netflix.titus.supplementary.relocation.RelocationConfiguration used on opensource projects

Example 1 with RelocationConfiguration

use of com.netflix.titus.supplementary.relocation.RelocationConfiguration in project titus-control-plane by Netflix.

the class DefaultNodeConditionControllerTest method checkTasksTerminatedDueToBadNodeConditions.

@Test
public void checkTasksTerminatedDueToBadNodeConditions() {
    // Mock jobs, tasks & nodes
    Map<String, TitusNode> nodeMap = buildNodes();
    List<Job<BatchJobExt>> jobs = getJobs(true);
    Map<String, List<Task>> tasksByJobIdMap = buildTasksForJobAndNodeAssignment(new ArrayList<>(nodeMap.values()), jobs);
    TitusRuntime titusRuntime = mock(TitusRuntime.class);
    when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
    RelocationConfiguration configuration = mock(RelocationConfiguration.class);
    when(configuration.getBadNodeConditionPattern()).thenReturn(".*Failure");
    when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
    NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
    when(nodeDataResolver.resolve()).thenReturn(nodeMap);
    JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
    when(jobDataReplicator.getStalenessMs()).thenReturn(0L);
    ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
    when(readOnlyJobOperations.getJobs()).thenReturn(new ArrayList<>(jobs));
    tasksByJobIdMap.forEach((key, value) -> when(readOnlyJobOperations.getTasks(key)).thenReturn(value));
    JobManagementClient jobManagementClient = mock(JobManagementClient.class);
    Set<String> terminatedTaskIds = new HashSet<>();
    when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
        String taskIdToBeTerminated = invocation.getArgument(0);
        terminatedTaskIds.add(taskIdToBeTerminated);
        return Mono.empty();
    });
    DefaultNodeConditionController nodeConditionCtrl = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
    ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
    StepVerifier.create(nodeConditionCtrl.handleNodesWithBadCondition(executionContext)).verifyComplete();
    assertThat(terminatedTaskIds).isNotEmpty();
    assertThat(terminatedTaskIds.size()).isEqualTo(2);
    verifyTerminatedTasksOnBadNodes(terminatedTaskIds, tasksByJobIdMap, nodeMap);
}

Example 2 with RelocationConfiguration

use of com.netflix.titus.supplementary.relocation.RelocationConfiguration in project titus-control-plane by Netflix.

the class DefaultNodeConditionControllerTest method noTerminationsOnDataStaleness.

@Test
public void noTerminationsOnDataStaleness() {
    TitusRuntime titusRuntime = mock(TitusRuntime.class);
    when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
    RelocationConfiguration configuration = mock(RelocationConfiguration.class);
    when(configuration.getBadNodeConditionPattern()).thenReturn(".*Problem");
    when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
    when(configuration.getDataStalenessThresholdMs()).thenReturn(8000L);
    NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
    when(nodeDataResolver.getStalenessMs()).thenReturn(5L);
    JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
    when(jobDataReplicator.getStalenessMs()).thenReturn(10L);
    ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
    JobManagementClient jobManagementClient = mock(JobManagementClient.class);
    Set<String> terminatedTaskIds = new HashSet<>();
    when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
        String taskIdToBeTerminated = invocation.getArgument(0);
        terminatedTaskIds.add(taskIdToBeTerminated);
        return Mono.empty();
    });
    DefaultNodeConditionController nodeConditionCtrl = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
    ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
    StepVerifier.create(nodeConditionCtrl.handleNodesWithBadCondition(executionContext)).verifyComplete();
    // No tasks terminated
    assertThat(terminatedTaskIds).isEmpty();
}

Example 3 with RelocationConfiguration

use of com.netflix.titus.supplementary.relocation.RelocationConfiguration in project titus-control-plane by Netflix.

the class DefaultNodeConditionControllerTest method badNodeConditionsIgnoredForJobsNotOptingIn.

@Test
public void badNodeConditionsIgnoredForJobsNotOptingIn() {
    Map<String, TitusNode> nodeMap = buildNodes();
    List<Job<BatchJobExt>> jobs = getJobs(false);
    Map<String, List<Task>> stringListMap = buildTasksForJobAndNodeAssignment(new ArrayList<>(nodeMap.values()), jobs);
    TitusRuntime titusRuntime = mock(TitusRuntime.class);
    when(titusRuntime.getRegistry()).thenReturn(new DefaultRegistry());
    RelocationConfiguration configuration = mock(RelocationConfiguration.class);
    when(configuration.getBadNodeConditionPattern()).thenReturn(".*Failure");
    when(configuration.isTaskTerminationOnBadNodeConditionEnabled()).thenReturn(true);
    NodeDataResolver nodeDataResolver = mock(NodeDataResolver.class);
    when(nodeDataResolver.resolve()).thenReturn(nodeMap);
    JobDataReplicator jobDataReplicator = mock(JobDataReplicator.class);
    when(jobDataReplicator.getStalenessMs()).thenReturn(0L);
    // Job attribute "terminateContainerOnBadAgent" = False
    ReadOnlyJobOperations readOnlyJobOperations = mock(ReadOnlyJobOperations.class);
    when(readOnlyJobOperations.getJobs()).thenReturn(new ArrayList<>(jobs));
    stringListMap.forEach((key, value) -> when(readOnlyJobOperations.getTasks(key)).thenReturn(value));
    JobManagementClient jobManagementClient = mock(JobManagementClient.class);
    Set<String> terminatedTaskIds = new HashSet<>();
    when(jobManagementClient.killTask(anyString(), anyBoolean(), any())).thenAnswer(invocation -> {
        String taskIdToBeTerminated = invocation.getArgument(0);
        terminatedTaskIds.add(taskIdToBeTerminated);
        return Mono.empty();
    });
    DefaultNodeConditionController nodeConditionController = new DefaultNodeConditionController(configuration, nodeDataResolver, jobDataReplicator, readOnlyJobOperations, jobManagementClient, titusRuntime);
    ExecutionContext executionContext = ExecutionContext.newBuilder().withIteration(ExecutionId.initial()).build();
    StepVerifier.create(nodeConditionController.handleNodesWithBadCondition(executionContext)).verifyComplete();
    // no tasks should be terminated for jobs
    assertThat(terminatedTaskIds).isEmpty();
}