use of com.netflix.titus.master.kubernetes.client.model.PodWrapper in project titus-control-plane by Netflix.
the class KubeNotificationProcessor method handlePodUpdatedEvent.
private Mono<Void> handlePodUpdatedEvent(PodEvent event, Job job, Task task) {
// This is basic sanity check. If it fails, we have a major problem with pod state.
if (event.getPod() == null || event.getPod().getStatus() == null || event.getPod().getStatus().getPhase() == null) {
logger.warn("Pod notification with pod without status or phase set: taskId={}, pod={}", task.getId(), event.getPod());
metricsNoChangesApplied.increment();
return Mono.empty();
}
PodWrapper podWrapper = new PodWrapper(event.getPod());
Optional<V1Node> node;
if (event instanceof PodUpdatedEvent) {
node = ((PodUpdatedEvent) event).getNode();
} else if (event instanceof PodDeletedEvent) {
node = ((PodDeletedEvent) event).getNode();
} else {
node = Optional.empty();
}
Either<TaskStatus, String> newTaskStatusOrError = new PodToTaskMapper(podWrapper, node, task, event instanceof PodDeletedEvent, containerResultCodeResolver, titusRuntime).getNewTaskStatus();
if (newTaskStatusOrError.hasError()) {
logger.info(newTaskStatusOrError.getError());
metricsNoChangesApplied.increment();
return Mono.empty();
}
TaskStatus newTaskStatus = newTaskStatusOrError.getValue();
if (TaskStatus.areEquivalent(task.getStatus(), newTaskStatus)) {
logger.info("Pod change notification does not change task status: taskId={}, status={}, eventSequenceNumber={}", task.getId(), newTaskStatus, event.getSequenceNumber());
} else {
logger.info("Pod notification changes task status: taskId={}, fromStatus={}, toStatus={}, eventSequenceNumber={}", task.getId(), task.getStatus(), newTaskStatus, event.getSequenceNumber());
}
// against most up to date task version.
if (!updateTaskStatus(podWrapper, newTaskStatus, node, task, true).isPresent()) {
return Mono.empty();
}
return ReactorExt.toMono(v3JobOperations.updateTask(task.getId(), current -> updateTaskStatus(podWrapper, newTaskStatus, node, current, false), V3JobOperations.Trigger.Kube, "Pod status updated from kubernetes node (k8phase='" + event.getPod().getStatus().getPhase() + "', taskState=" + task.getStatus().getState() + ")", KUBE_CALL_METADATA));
}
use of com.netflix.titus.master.kubernetes.client.model.PodWrapper in project titus-control-plane by Netflix.
the class KubeNotificationProcessorTest method testUpdateTaskStatusVK.
@Test
public void testUpdateTaskStatusVK() {
V1Pod pod = newPod(TASK.getId(), andRunning());
V1Node node = newNode(andIpAddress("2.2.2.2"), andNodeAnnotations(TITUS_NODE_DOMAIN + "ami", "ami123", TITUS_NODE_DOMAIN + "stack", "myStack"));
Map<String, String> UpdatedAnnotations = new HashMap<>();
UpdatedAnnotations.put(LEGACY_ANNOTATION_IP_ADDRESS, "1.2.3.4");
pod.getMetadata().setAnnotations(UpdatedAnnotations);
Task updatedTask = processor.updateTaskStatus(new PodWrapper(pod), TaskStatus.newBuilder().withState(TaskState.Started).build(), Optional.of(node), TASK, false).orElse(null);
Set<TaskState> pastStates = updatedTask.getStatusHistory().stream().map(ExecutableStatus::getState).collect(Collectors.toSet());
assertThat(pastStates).contains(TaskState.Accepted, TaskState.Launched, TaskState.StartInitiated);
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_AGENT_HOST, "2.2.2.2");
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "1.2.3.4");
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_AGENT_AMI, "ami123");
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_AGENT_STACK, "myStack");
}
use of com.netflix.titus.master.kubernetes.client.model.PodWrapper in project titus-control-plane by Netflix.
the class KubeNotificationProcessor method updateTaskStatus.
@VisibleForTesting
Optional<Task> updateTaskStatus(PodWrapper podWrapper, TaskStatus newTaskStatus, Optional<V1Node> node, Task currentTask, boolean precheck) {
// 8. in the next reconciliation loop a task is moved again to 'KillInitiated' state.
if (TaskState.isBefore(newTaskStatus.getState(), currentTask.getStatus().getState())) {
logger.info("[precheck={}] Ignoring an attempt to move the task state to the earlier one: taskId={}, attempt={}, current={}", precheck, currentTask.getId(), newTaskStatus.getState(), currentTask.getStatus().getState());
metricsNoChangesApplied.increment();
return Optional.empty();
}
Task updatedTask;
if (TaskStatus.areEquivalent(currentTask.getStatus(), newTaskStatus)) {
updatedTask = currentTask;
} else {
List<TaskStatus> newHistory = CollectionsExt.copyAndAdd(currentTask.getStatusHistory(), currentTask.getStatus());
updatedTask = currentTask.toBuilder().withStatus(newTaskStatus).withStatusHistory(newHistory).build();
}
Task fixedTask = fillInMissingStates(podWrapper, updatedTask);
Task taskWithExecutorData = JobManagerUtil.attachNetworkDataFromPod(fixedTask, podWrapper);
Task taskWithNodeMetadata = node.map(n -> attachNodeMetadata(taskWithExecutorData, n)).orElse(taskWithExecutorData);
Optional<String> difference = areTasksEquivalent(currentTask, taskWithNodeMetadata);
if (!difference.isPresent()) {
logger.debug("[precheck={}] Ignoring the pod event as the update results in the identical task object as the current one: taskId={}", precheck, currentTask.getId());
metricsNoChangesApplied.increment();
return Optional.empty();
}
if (!precheck) {
logger.info("[precheck={}] Tasks are different: difference='{}', current={}, updated={}", precheck, difference.get(), currentTask, taskWithNodeMetadata);
metricsChangesApplied.increment();
}
return Optional.of(taskWithNodeMetadata);
}
use of com.netflix.titus.master.kubernetes.client.model.PodWrapper in project titus-control-plane by Netflix.
the class KubeNotificationProcessorTest method testTaskStateDoesNotMoveBack.
@Test
public void testTaskStateDoesNotMoveBack() {
V1Pod pod = newPod(TASK.getId(), andRunning());
Task updatedTask = processor.updateTaskStatus(new PodWrapper(pod), TaskStatus.newBuilder().withState(TaskState.Started).build(), Optional.of(newNode()), JobFunctions.changeTaskStatus(TASK, TaskStatus.newBuilder().withState(TaskState.KillInitiated).build()), false).orElse(null);
assertThat(updatedTask).isNull();
}
use of com.netflix.titus.master.kubernetes.client.model.PodWrapper in project titus-control-plane by Netflix.
the class KubeNotificationProcessorTest method testUpdateTaskStatusVKWithTransitionNetworkMode.
@Test
public void testUpdateTaskStatusVKWithTransitionNetworkMode() {
V1Pod pod = newPod(TASK.getId(), andRunning());
V1Node node = newNode(andIpAddress("2.2.2.2"), andNodeAnnotations(TITUS_NODE_DOMAIN + "ami", "ami123", TITUS_NODE_DOMAIN + "stack", "myStack"));
Map<String, String> UpdatedAnnotations = new HashMap<>();
UpdatedAnnotations.put(LEGACY_ANNOTATION_IP_ADDRESS, "2001:db8:0:1234:0:567:8:1");
UpdatedAnnotations.put(LEGACY_ANNOTATION_ENI_IP_ADDRESS, "192.0.2.1");
UpdatedAnnotations.put(LEGACY_ANNOTATION_ENI_IPV6_ADDRESS, "2001:db8:0:1234:0:567:8:1");
UpdatedAnnotations.put(LEGACY_ANNOTATION_NETWORK_MODE, NetworkConfiguration.NetworkMode.Ipv6AndIpv4Fallback.toString());
pod.getMetadata().setAnnotations(UpdatedAnnotations);
Task updatedTask = processor.updateTaskStatus(new PodWrapper(pod), TaskStatus.newBuilder().withState(TaskState.Started).build(), Optional.of(node), TASK, false).orElse(null);
Set<TaskState> pastStates = updatedTask.getStatusHistory().stream().map(ExecutableStatus::getState).collect(Collectors.toSet());
assertThat(pastStates).contains(TaskState.Accepted, TaskState.Launched, TaskState.StartInitiated);
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_AGENT_HOST, "2.2.2.2");
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "2001:db8:0:1234:0:567:8:1");
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IPV6, "2001:db8:0:1234:0:567:8:1");
// In IPv6 + transition mode, there should *not* be a ipv4. That would be confusing because such a v4 would not
// be unique to that task, and tools would try to use it, people would try to ssh to it, etc.
assertThat(updatedTask.getTaskContext()).doesNotContainKey(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IPV4);
assertThat(updatedTask.getTaskContext()).containsEntry(TaskAttributes.TASK_ATTRIBUTES_TRANSITION_IPV4, "192.0.2.1");
}
Aggregations