use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.
the class DefaultLoadBalancerReconcilerTest method connectorErrorsDoNotHaltReconciliation.
@Test(timeout = TEST_TIMEOUT_MS)
public void connectorErrorsDoNotHaltReconciliation() {
String failingLoadBalancerId = UUID.randomUUID().toString();
List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
JobLoadBalancerState association = new JobLoadBalancerState(jobLoadBalancer, State.ASSOCIATED);
JobLoadBalancer failingJobLoadBalancer = new JobLoadBalancer(jobId, failingLoadBalancerId);
JobLoadBalancerState failingAssociation = new JobLoadBalancerState(failingJobLoadBalancer, State.ASSOCIATED);
when(v3JobOperations.getTasks(jobId)).thenReturn(tasks);
when(connector.getLoadBalancer(failingLoadBalancerId)).thenReturn(Single.error(new RuntimeException("rate limit")));
Completable.merge(store.addOrUpdateLoadBalancer(failingAssociation.getJobLoadBalancer(), failingAssociation.getState()), store.addOrUpdateLoadBalancer(association.getJobLoadBalancer(), association.getState())).await();
testScheduler.triggerActions();
subscriber.assertNoErrors().assertNotCompleted().assertNoValues();
awaitReconciliationRuns(1);
// failingLoadBalancerId gets ignored
subscriber.assertNoErrors().assertNotCompleted().assertValueCount(5);
subscriber.getOnNextEvents().forEach(update -> {
assertThat(update.getState()).isEqualTo(LoadBalancerTarget.State.REGISTERED);
assertThat(update.getPriority()).isEqualTo(Priority.LOW);
assertThat(update.getLoadBalancerId()).isEqualTo(loadBalancerId);
});
}
use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.
the class DefaultLoadBalancerReconcilerTest method orphanLoadBalancerAssociationsAreSetAsDissociatedAndRemoved.
@Test
public void orphanLoadBalancerAssociationsAreSetAsDissociatedAndRemoved() {
List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
when(v3JobOperations.getTasks(jobId)).thenReturn(tasks);
reset(connector);
OngoingStubbing<Single<LoadBalancer>> ongoingStubbing = when(connector.getLoadBalancer(loadBalancerId)).thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.ACTIVE, CollectionsExt.asSet("1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4", "5.5.5.5"))));
assertThat(store.addOrUpdateLoadBalancer(jobLoadBalancer, State.ASSOCIATED).await(5, TimeUnit.SECONDS)).isTrue();
// all targets were previously registered by us
store.addOrUpdateTargets(tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), LoadBalancerTarget.State.REGISTERED)).collect(Collectors.toList())).block();
testScheduler.triggerActions();
subscriber.assertNotCompleted().assertNoValues();
// load balancer was removed outside of Titus
ongoingStubbing.thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.REMOVED, Collections.emptySet())));
// Let a few iterations run so all phases can be executed:
// 1. mark as orphan
// 2. update targets as DEREGISTERED
awaitReconciliationRuns(2);
subscriber.awaitValueCount(5, TEST_TIMEOUT_MS / 2, TimeUnit.MILLISECONDS).assertNoErrors();
assertThat(subscriber.getOnNextEvents()).allMatch(update -> update.getState().equals(LoadBalancerTarget.State.DEREGISTERED));
// simulate all targets got DEREGISTERED
List<LoadBalancerTargetState> currentTargets = store.getLoadBalancerTargets(loadBalancerId).collectList().block();
assertThat(currentTargets).isNotNull();
store.addOrUpdateTargets(currentTargets.stream().map(targetState -> targetState.getLoadBalancerTarget().withState(LoadBalancerTarget.State.DEREGISTERED)).collect(Collectors.toList())).block();
// 3. update orphan as Dissociated
awaitReconciliationRuns(1);
assertThat(store.getAssociations()).containsOnly(new JobLoadBalancerState(jobLoadBalancer, State.DISSOCIATED));
// 4. sweep all targets
// 5. sweep all Dissociated
awaitReconciliationRuns(2);
assertThat(store.getLoadBalancerTargets(loadBalancerId).collectList().block()).isEmpty();
assertThat(store.getAssociations()).isEmpty();
assertThat(store.getAssociatedLoadBalancersSetForJob(jobId)).isEmpty();
}
use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.
the class DefaultLoadBalancerReconcilerTest method jobsWithErrorsAreIgnored.
@Test(timeout = TEST_TIMEOUT_MS)
public void jobsWithErrorsAreIgnored() {
List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
JobLoadBalancerState association = new JobLoadBalancerState(jobLoadBalancer, State.ASSOCIATED);
when(v3JobOperations.getTasks(jobId)).thenThrow(// first fails
JobManagerException.class).thenReturn(tasks);
store.addOrUpdateLoadBalancer(association.getJobLoadBalancer(), association.getState()).await();
testScheduler.triggerActions();
subscriber.assertNotCompleted().assertNoValues();
awaitReconciliationRuns(1);
subscriber.assertNotCompleted().assertNoValues();
awaitReconciliationRuns(1);
subscriber.assertNotCompleted().assertValueCount(5);
subscriber.getOnNextEvents().forEach(update -> {
assertThat(update.getState()).isEqualTo(LoadBalancerTarget.State.REGISTERED);
assertThat(update.getPriority()).isEqualTo(Priority.LOW);
assertThat(update.getLoadBalancerId()).isEqualTo(loadBalancerId);
});
}
use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.
the class DefaultLoadBalancerReconcilerTest method dissociatedJobsAreNotRemovedUntilAllTargetsAreDeregisteredAndRemoved.
@Test
public void dissociatedJobsAreNotRemovedUntilAllTargetsAreDeregisteredAndRemoved() throws InterruptedException {
JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
when(v3JobOperations.getTasks(jobId)).thenThrow(JobManagerException.jobNotFound(jobId));
when(v3JobOperations.getJob(jobId)).thenReturn(Optional.empty());
reset(connector);
when(connector.getLoadBalancer(loadBalancerId)).thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.ACTIVE, Collections.singleton("1.2.3.4"))));
store.addOrUpdateTargets(new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, "some-task", "1.2.3.4"), LoadBalancerTarget.State.DEREGISTERED)).block();
assertThat(store.addOrUpdateLoadBalancer(jobLoadBalancer, State.DISSOCIATED).await(5, TimeUnit.SECONDS)).isTrue();
testScheduler.triggerActions();
subscriber.assertNotCompleted().assertNoValues();
// 1. deregister
awaitReconciliationRuns(1);
subscriber.assertNoTerminalEvent().assertValueCount(1);
assertThat(store.getAssociations()).isNotEmpty().hasSize(1);
when(connector.getLoadBalancer(loadBalancerId)).thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.ACTIVE, Collections.emptySet())));
// Let a few iterations run so the remaining phases have a chance to complete:
// 2. clean up target state
// 3. clean up association
awaitReconciliationRuns(3);
assertThat(store.getAssociations()).isEmpty();
assertThat(store.getLoadBalancerTargets(loadBalancerId).collectList().block()).isEmpty();
}
use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.
the class DefaultLoadBalancerServiceTest method backfillsCurrentTargetsToStore.
@Test(timeout = 30_000)
public void backfillsCurrentTargetsToStore() {
String jobId = UUID.randomUUID().toString();
String associatedId = "lb-" + UUID.randomUUID().toString();
String dissociatedId = "lb-" + UUID.randomUUID().toString();
String removedId = "lb-" + UUID.randomUUID().toString();
LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
when(configuration.isTargetsToStoreBackfillEnabled()).thenReturn(true);
when(configuration.getStoreBackfillConcurrencyLimit()).thenReturn(10);
when(configuration.getStoreBackfillTimeoutMs()).thenReturn(5000L);
// current load balancer state (targets)
when(client.getLoadBalancer(associatedId)).thenReturn(Single.just(new LoadBalancer(associatedId, LoadBalancer.State.ACTIVE, CollectionsExt.asSet("1.1.1.1", "2.2.2.2", "3.3.3.3"))));
when(client.getLoadBalancer(dissociatedId)).thenReturn(Single.just(new LoadBalancer(dissociatedId, LoadBalancer.State.ACTIVE, CollectionsExt.asSet("4.4.4.4", "5.5.5.5", "6.6.6.6"))));
when(client.getLoadBalancer(removedId)).thenReturn(Single.just(new LoadBalancer(removedId, LoadBalancer.State.REMOVED, Collections.emptySet())));
// current load balancers we are managing
loadBalancerStore.addOrUpdateLoadBalancer(new JobLoadBalancer(jobId, associatedId), JobLoadBalancer.State.ASSOCIATED).await();
loadBalancerStore.addOrUpdateLoadBalancer(new JobLoadBalancer(jobId, dissociatedId), JobLoadBalancer.State.DISSOCIATED).await();
loadBalancerStore.addOrUpdateLoadBalancer(new JobLoadBalancer(jobId, removedId), JobLoadBalancer.State.ASSOCIATED).await();
DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
service.backfillTargetsToStore();
assertThat(loadBalancerStore.getLoadBalancerTargets(associatedId).collectList().block()).containsExactlyInAnyOrder(new LoadBalancerTargetState(new LoadBalancerTarget(associatedId, "BACKFILLED", "1.1.1.1"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(associatedId, "BACKFILLED", "2.2.2.2"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(associatedId, "BACKFILLED", "3.3.3.3"), REGISTERED));
assertThat(loadBalancerStore.getLoadBalancerTargets(dissociatedId).collectList().block()).containsExactlyInAnyOrder(new LoadBalancerTargetState(new LoadBalancerTarget(dissociatedId, "BACKFILLED", "4.4.4.4"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(dissociatedId, "BACKFILLED", "5.5.5.5"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(dissociatedId, "BACKFILLED", "6.6.6.6"), REGISTERED));
assertThat(loadBalancerStore.getLoadBalancerTargets(removedId).collectList().block()).isEmpty();
}
Aggregations