use of com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState in project titus-control-plane by Netflix.
the class DefaultLoadBalancerServiceTest method finishedTasksGetDeregistered.
@Test
public void finishedTasksGetDeregistered() {
String jobId = UUID.randomUUID().toString();
String loadBalancerId = "lb-" + UUID.randomUUID().toString();
PublishSubject<JobManagerEvent<?>> taskEvents = PublishSubject.create();
when(client.registerAll(any(), any())).thenReturn(Completable.complete());
when(client.deregisterAll(any(), any())).thenReturn(Completable.complete());
when(v3JobOperations.observeJobs()).thenReturn(taskEvents);
when(v3JobOperations.getTasks(jobId)).thenReturn(Collections.emptyList());
LoadBalancerTests.applyValidGetJobMock(v3JobOperations, jobId);
LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
assertTrue(service.addLoadBalancer(jobId, loadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertThat(service.getJobLoadBalancers(jobId).toBlocking().first()).isEqualTo(loadBalancerId);
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(0);
verify(client, never()).registerAll(any(), any());
verify(client, never()).deregisterAll(any(), any());
verifyNoReconcilerIgnore();
// a task that was prematurely killed before having an IP address associated to it should be ignored
Task noIp = ServiceJobTask.newBuilder().withJobId(jobId).withId(UUID.randomUUID().toString()).withStatus(TaskStatus.newBuilder().withState(TaskState.KillInitiated).build()).build();
Task noIpFinished = noIp.toBuilder().withStatus(TaskStatus.newBuilder().withState(TaskState.Finished).build()).build();
taskEvents.onNext(TaskUpdateEvent.taskChange(null, noIpFinished, noIp, callMetadata));
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(0);
verify(client, never()).registerAll(any(), any());
verify(client, never()).deregisterAll(any(), any());
verifyNoReconcilerIgnore();
// 3 state transitions to 3 different terminal events
Task first = noIp.toBuilder().withId(UUID.randomUUID().toString()).withStatus(TaskStatus.newBuilder().withState(TaskState.Started).build()).withTaskContext(CollectionsExt.asMap(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "1.1.1.1")).build();
Task firstFinished = first.toBuilder().withStatus(TaskStatus.newBuilder().withState(TaskState.Finished).build()).build();
LoadBalancerTargetState expectedFirstTarget = new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, firstFinished.getId(), "1.1.1.1"), DEREGISTERED);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).doesNotContain(expectedFirstTarget);
taskEvents.onNext(TaskUpdateEvent.taskChange(null, firstFinished, first, callMetadata));
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(1);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).contains(expectedFirstTarget);
verify(client, never()).registerAll(eq(loadBalancerId), any());
verify(client).deregisterAll(eq(loadBalancerId), argThat(set -> set.contains("1.1.1.1")));
verifyReconcilerIgnore(loadBalancerId, "1.1.1.1");
Task second = first.toBuilder().withId(UUID.randomUUID().toString()).withTaskContext(CollectionsExt.asMap(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "2.2.2.2")).build();
Task secondKilling = second.toBuilder().withStatus(TaskStatus.newBuilder().withState(TaskState.KillInitiated).build()).build();
LoadBalancerTargetState expectedSecondTarget = new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, secondKilling.getId(), "2.2.2.2"), DEREGISTERED);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).doesNotContain(expectedSecondTarget);
taskEvents.onNext(TaskUpdateEvent.taskChange(null, secondKilling, second, callMetadata));
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(2);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).contains(expectedSecondTarget);
verify(client, never()).registerAll(eq(loadBalancerId), any());
verify(client).deregisterAll(eq(loadBalancerId), argThat(set -> set.contains("2.2.2.2")));
verifyReconcilerIgnore(loadBalancerId, "2.2.2.2");
Task third = first.toBuilder().withId(UUID.randomUUID().toString()).withTaskContext(CollectionsExt.asMap(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "3.3.3.3")).build();
Task thirdDisconnected = third.toBuilder().withStatus(TaskStatus.newBuilder().withState(TaskState.Disconnected).build()).build();
LoadBalancerTargetState expectedThirdTarget = new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, thirdDisconnected.getId(), "3.3.3.3"), DEREGISTERED);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).doesNotContain(expectedThirdTarget);
taskEvents.onNext(TaskUpdateEvent.taskChange(null, thirdDisconnected, third, callMetadata));
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(3);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).contains(expectedThirdTarget);
verify(client, never()).registerAll(eq(loadBalancerId), any());
verify(client).deregisterAll(eq(loadBalancerId), argThat(set -> set.contains("3.3.3.3")));
verifyReconcilerIgnore(loadBalancerId, "3.3.3.3");
}
use of com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState in project titus-control-plane by Netflix.
the class DefaultLoadBalancerServiceTest method addLoadBalancerRegistersTasks.
@Test
public void addLoadBalancerRegistersTasks() {
String jobId = UUID.randomUUID().toString();
String loadBalancerId = "lb-" + UUID.randomUUID().toString();
defaultStubs();
LoadBalancerTests.applyValidGetJobMock(v3JobOperations, jobId);
List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
Collection<LoadBalancerTargetState> expectedTargets = tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), REGISTERED)).collect(Collectors.toList());
when(v3JobOperations.getTasks(jobId)).thenReturn(CollectionsExt.merge(tasks, LoadBalancerTests.buildTasks(2, jobId, TaskState.StartInitiated), LoadBalancerTests.buildTasks(2, jobId, TaskState.KillInitiated), LoadBalancerTests.buildTasks(3, jobId, TaskState.Finished), LoadBalancerTests.buildTasks(1, jobId, TaskState.Disconnected)));
LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
assertTrue(service.addLoadBalancer(jobId, loadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertThat(service.getJobLoadBalancers(jobId).toBlocking().first()).isEqualTo(loadBalancerId);
verify(v3JobOperations).getTasks(jobId);
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(1);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).containsExactlyInAnyOrderElementsOf(expectedTargets);
verify(client).registerAll(eq(loadBalancerId), argThat(targets -> targets != null && targets.size() == tasks.size()));
verify(client, never()).deregisterAll(eq(loadBalancerId), any());
verifyReconcilerIgnore(loadBalancerId, LoadBalancerTests.ipAddresses(tasks));
}
use of com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState in project titus-control-plane by Netflix.
the class DefaultLoadBalancerReconciler method updatesForRemovedLoadBalancer.
private ReconciliationUpdates updatesForRemovedLoadBalancer(LoadBalancerWithKnownTargets loadBalancer, List<JobLoadBalancerState> associations) {
logger.warn("Load balancer is gone, ignoring its associations (marking them to be GCed later), and removing known state for its targets: {}", loadBalancer.current.getId());
for (JobLoadBalancerState association : associations) {
if (association.isStateAssociated()) {
logger.info("Marking association as orphan: {}", association.getJobLoadBalancer());
markedAsOrphan.add(association.getJobLoadBalancer());
}
}
// we will force deregistration of everything that was marked as REGISTERED to force its state to be updated
Map<Boolean, Set<LoadBalancerTarget>> registeredOrNot = loadBalancer.knownTargets.stream().collect(partitioningBy(target -> target.getState().equals(State.REGISTERED), mapping(LoadBalancerTargetState::getLoadBalancerTarget, Collectors.toSet())));
// all REGISTERED
Set<LoadBalancerTarget> toDeregister = registeredOrNot.get(true);
// all DEREGISTERED
Set<LoadBalancerTarget> toRemove = registeredOrNot.get(false);
return new ReconciliationUpdates(loadBalancer.current.getId(), Collections.emptySet(), toDeregister, toRemove);
}
use of com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState in project titus-control-plane by Netflix.
the class CassandraLoadBalancerStoreTest method testOnlyDeregisteredTargetsAreRemoved.
@Test
public void testOnlyDeregisteredTargetsAreRemoved() throws Exception {
Map<LoadBalancerTarget, LoadBalancerTarget.State> targets = ImmutableMap.of(new LoadBalancerTarget("lb-1", "task1", "1.1.1.1"), LoadBalancerTarget.State.REGISTERED, new LoadBalancerTarget("lb-1", "task2", "2.2.2.2"), LoadBalancerTarget.State.DEREGISTERED, new LoadBalancerTarget("lb-2", "task1", "1.1.1.1"), LoadBalancerTarget.State.DEREGISTERED, new LoadBalancerTarget("lb-2", "task3", "3.3.3.3"), LoadBalancerTarget.State.DEREGISTERED);
loadTestData(new TestData(Collections.emptyMap(), targets));
CassandraLoadBalancerStore store = getInitdStore();
store.removeDeregisteredTargets(targets.keySet()).block(Duration.ofSeconds(10));
List<LoadBalancerTargetState> targets1 = store.getLoadBalancerTargets("lb-1").collectList().block(Duration.ofSeconds(5));
assertThat(targets1).hasSize(1);
assertThat(targets1.get(0).getIpAddress()).isEqualTo("1.1.1.1");
List<LoadBalancerTargetState> targets2 = store.getLoadBalancerTargets("lb-2").collectList().block(Duration.ofSeconds(5));
assertThat(targets2).isEmpty();
}
use of com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState in project titus-control-plane by Netflix.
the class DefaultLoadBalancerServiceTest method multipleLoadBalancersPerJob.
@Test
public void multipleLoadBalancersPerJob() {
PublishSubject<JobManagerEvent<?>> taskEvents = PublishSubject.create();
String jobId = UUID.randomUUID().toString();
String firstLoadBalancerId = "lb-" + UUID.randomUUID().toString();
String secondLoadBalancerId = "lb-" + UUID.randomUUID().toString();
int numberOfStartedTasks = 5;
when(client.registerAll(any(), any())).thenReturn(Completable.complete());
when(client.deregisterAll(any(), any())).thenReturn(Completable.complete());
when(v3JobOperations.observeJobs()).thenReturn(taskEvents);
LoadBalancerTests.applyValidGetJobMock(v3JobOperations, jobId);
List<Task> tasks = LoadBalancerTests.buildTasksStarted(numberOfStartedTasks, jobId);
Collection<LoadBalancerTargetState> firstExpectedTargets = tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(firstLoadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), REGISTERED)).collect(Collectors.toList());
Collection<LoadBalancerTargetState> secondExpectedTargets = tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(secondLoadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), REGISTERED)).collect(Collectors.toList());
when(v3JobOperations.getTasks(jobId)).thenReturn(CollectionsExt.merge(tasks, LoadBalancerTests.buildTasks(2, jobId, TaskState.StartInitiated), LoadBalancerTests.buildTasks(2, jobId, TaskState.KillInitiated), LoadBalancerTests.buildTasks(3, jobId, TaskState.Finished), LoadBalancerTests.buildTasks(1, jobId, TaskState.Disconnected)));
LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
// associate two load balancers to the same job
assertTrue(service.addLoadBalancer(jobId, firstLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertTrue(service.addLoadBalancer(jobId, secondLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertThat(service.getJobLoadBalancers(jobId).toList().toBlocking().single()).containsOnly(firstLoadBalancerId, secondLoadBalancerId);
verify(v3JobOperations, times(2)).getTasks(jobId);
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
// 1 batch per loadbalancer
testSubscriber.assertNoErrors().assertValueCount(2);
assertThat(loadBalancerStore.getLoadBalancerTargets(firstLoadBalancerId).collectList().block()).containsExactlyInAnyOrderElementsOf(firstExpectedTargets);
assertThat(loadBalancerStore.getLoadBalancerTargets(secondLoadBalancerId).collectList().block()).containsExactlyInAnyOrderElementsOf(secondExpectedTargets);
verify(client).registerAll(eq(firstLoadBalancerId), argThat(targets -> targets != null && targets.size() == numberOfStartedTasks));
verify(client).registerAll(eq(secondLoadBalancerId), argThat(targets -> targets != null && targets.size() == numberOfStartedTasks));
verify(client, never()).deregisterAll(eq(firstLoadBalancerId), any());
verify(client, never()).deregisterAll(eq(secondLoadBalancerId), any());
verifyReconcilerIgnore(firstLoadBalancerId, LoadBalancerTests.ipAddresses(tasks));
verifyReconcilerIgnore(secondLoadBalancerId, LoadBalancerTests.ipAddresses(tasks));
// now some more tasks are added to the job, check if both load balancers get updated
List<Task> newTasks = new ArrayList<>();
for (int i = 1; i <= numberOfStartedTasks; i++) {
String taskId = UUID.randomUUID().toString();
Task startingWithIp = ServiceJobTask.newBuilder().withJobId(jobId).withId(taskId).withStatus(TaskStatus.newBuilder().withState(TaskState.StartInitiated).build()).withTaskContext(CollectionsExt.asMap(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, String.format("%1$d.%1$d.%1$d.%1$d", i + numberOfStartedTasks))).build();
Task started = startingWithIp.toBuilder().withStatus(TaskStatus.newBuilder().withState(TaskState.Started).build()).build();
newTasks.add(started);
taskEvents.onNext(TaskUpdateEvent.taskChange(null, started, startingWithIp, callMetadata));
}
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
// 2 more batches (one per load balancer)
testSubscriber.assertNoErrors().assertValueCount(4);
verify(client, times(2)).registerAll(eq(firstLoadBalancerId), argThat(targets -> targets != null && targets.size() == numberOfStartedTasks));
verify(client, times(2)).registerAll(eq(secondLoadBalancerId), argThat(targets -> targets != null && targets.size() == numberOfStartedTasks));
verify(client, never()).deregisterAll(eq(firstLoadBalancerId), any());
verify(client, never()).deregisterAll(eq(secondLoadBalancerId), any());
verifyReconcilerIgnore(firstLoadBalancerId, LoadBalancerTests.ipAddresses(newTasks));
verifyReconcilerIgnore(secondLoadBalancerId, LoadBalancerTests.ipAddresses(newTasks));
assertThat(loadBalancerStore.getLoadBalancerTargets(firstLoadBalancerId).collectList().block()).hasSize(firstExpectedTargets.size() + numberOfStartedTasks);
assertThat(loadBalancerStore.getLoadBalancerTargets(secondLoadBalancerId).collectList().block()).hasSize(secondExpectedTargets.size() + numberOfStartedTasks);
}
Aggregations