Search in sources :

Example 31 with JobLoadBalancer

use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.

the class DefaultLoadBalancerReconcilerTest method connectorErrorsDoNotHaltReconciliation.

@Test(timeout = TEST_TIMEOUT_MS)
public void connectorErrorsDoNotHaltReconciliation() {
    String failingLoadBalancerId = UUID.randomUUID().toString();
    List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
    JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
    JobLoadBalancerState association = new JobLoadBalancerState(jobLoadBalancer, State.ASSOCIATED);
    JobLoadBalancer failingJobLoadBalancer = new JobLoadBalancer(jobId, failingLoadBalancerId);
    JobLoadBalancerState failingAssociation = new JobLoadBalancerState(failingJobLoadBalancer, State.ASSOCIATED);
    when(v3JobOperations.getTasks(jobId)).thenReturn(tasks);
    when(connector.getLoadBalancer(failingLoadBalancerId)).thenReturn(Single.error(new RuntimeException("rate limit")));
    Completable.merge(store.addOrUpdateLoadBalancer(failingAssociation.getJobLoadBalancer(), failingAssociation.getState()), store.addOrUpdateLoadBalancer(association.getJobLoadBalancer(), association.getState())).await();
    testScheduler.triggerActions();
    subscriber.assertNoErrors().assertNotCompleted().assertNoValues();
    awaitReconciliationRuns(1);
    // failingLoadBalancerId gets ignored
    subscriber.assertNoErrors().assertNotCompleted().assertValueCount(5);
    subscriber.getOnNextEvents().forEach(update -> {
        assertThat(update.getState()).isEqualTo(LoadBalancerTarget.State.REGISTERED);
        assertThat(update.getPriority()).isEqualTo(Priority.LOW);
        assertThat(update.getLoadBalancerId()).isEqualTo(loadBalancerId);
    });
}
Also used : Task(com.netflix.titus.api.jobmanager.model.job.Task) JobLoadBalancerState(com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) Test(org.junit.Test)

Example 32 with JobLoadBalancer

use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.

the class DefaultLoadBalancerReconcilerTest method orphanLoadBalancerAssociationsAreSetAsDissociatedAndRemoved.

@Test
public void orphanLoadBalancerAssociationsAreSetAsDissociatedAndRemoved() {
    List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
    JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
    when(v3JobOperations.getTasks(jobId)).thenReturn(tasks);
    reset(connector);
    OngoingStubbing<Single<LoadBalancer>> ongoingStubbing = when(connector.getLoadBalancer(loadBalancerId)).thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.ACTIVE, CollectionsExt.asSet("1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4", "5.5.5.5"))));
    assertThat(store.addOrUpdateLoadBalancer(jobLoadBalancer, State.ASSOCIATED).await(5, TimeUnit.SECONDS)).isTrue();
    // all targets were previously registered by us
    store.addOrUpdateTargets(tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), LoadBalancerTarget.State.REGISTERED)).collect(Collectors.toList())).block();
    testScheduler.triggerActions();
    subscriber.assertNotCompleted().assertNoValues();
    // load balancer was removed outside of Titus
    ongoingStubbing.thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.REMOVED, Collections.emptySet())));
    // Let a few iterations run so all phases can be executed:
    // 1. mark as orphan
    // 2. update targets as DEREGISTERED
    awaitReconciliationRuns(2);
    subscriber.awaitValueCount(5, TEST_TIMEOUT_MS / 2, TimeUnit.MILLISECONDS).assertNoErrors();
    assertThat(subscriber.getOnNextEvents()).allMatch(update -> update.getState().equals(LoadBalancerTarget.State.DEREGISTERED));
    // simulate all targets got DEREGISTERED
    List<LoadBalancerTargetState> currentTargets = store.getLoadBalancerTargets(loadBalancerId).collectList().block();
    assertThat(currentTargets).isNotNull();
    store.addOrUpdateTargets(currentTargets.stream().map(targetState -> targetState.getLoadBalancerTarget().withState(LoadBalancerTarget.State.DEREGISTERED)).collect(Collectors.toList())).block();
    // 3. update orphan as Dissociated
    awaitReconciliationRuns(1);
    assertThat(store.getAssociations()).containsOnly(new JobLoadBalancerState(jobLoadBalancer, State.DISSOCIATED));
    // 4. sweep all targets
    // 5. sweep all Dissociated
    awaitReconciliationRuns(2);
    assertThat(store.getLoadBalancerTargets(loadBalancerId).collectList().block()).isEmpty();
    assertThat(store.getAssociations()).isEmpty();
    assertThat(store.getAssociatedLoadBalancersSetForJob(jobId)).isEmpty();
}
Also used : NoopRegistry(com.netflix.spectator.api.NoopRegistry) Completable(rx.Completable) LoadBalancerConnector(com.netflix.titus.api.connector.cloud.LoadBalancerConnector) JobLoadBalancerState(com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) Task(com.netflix.titus.api.jobmanager.model.job.Task) CollectionsExt(com.netflix.titus.common.util.CollectionsExt) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TestScheduler(rx.schedulers.TestScheduler) State(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer.State) AssertableSubscriber(rx.observers.AssertableSubscriber) Single(rx.Single) After(org.junit.After) Schedulers(rx.schedulers.Schedulers) JobManagerException(com.netflix.titus.api.jobmanager.service.JobManagerException) Before(org.junit.Before) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) LoadBalancerStore(com.netflix.titus.api.loadbalancer.store.LoadBalancerStore) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) Priority(com.netflix.titus.common.util.rx.batch.Priority) Awaitility.await(com.jayway.awaitility.Awaitility.await) OngoingStubbing(org.mockito.stubbing.OngoingStubbing) Test(org.junit.Test) UUID(java.util.UUID) Mockito.when(org.mockito.Mockito.when) LoadBalancer(com.netflix.titus.api.connector.cloud.LoadBalancer) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) TaskAttributes(com.netflix.titus.api.jobmanager.TaskAttributes) InMemoryLoadBalancerStore(com.netflix.titus.runtime.store.v3.memory.InMemoryLoadBalancerStore) Optional(java.util.Optional) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Collections(java.util.Collections) Mockito.reset(org.mockito.Mockito.reset) Mockito.mock(org.mockito.Mockito.mock) Task(com.netflix.titus.api.jobmanager.model.job.Task) JobLoadBalancerState(com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState) Single(rx.Single) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancer(com.netflix.titus.api.connector.cloud.LoadBalancer) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) Test(org.junit.Test)

Example 33 with JobLoadBalancer

use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.

the class DefaultLoadBalancerReconcilerTest method jobsWithErrorsAreIgnored.

@Test(timeout = TEST_TIMEOUT_MS)
public void jobsWithErrorsAreIgnored() {
    List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
    JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
    JobLoadBalancerState association = new JobLoadBalancerState(jobLoadBalancer, State.ASSOCIATED);
    when(v3JobOperations.getTasks(jobId)).thenThrow(// first fails
    JobManagerException.class).thenReturn(tasks);
    store.addOrUpdateLoadBalancer(association.getJobLoadBalancer(), association.getState()).await();
    testScheduler.triggerActions();
    subscriber.assertNotCompleted().assertNoValues();
    awaitReconciliationRuns(1);
    subscriber.assertNotCompleted().assertNoValues();
    awaitReconciliationRuns(1);
    subscriber.assertNotCompleted().assertValueCount(5);
    subscriber.getOnNextEvents().forEach(update -> {
        assertThat(update.getState()).isEqualTo(LoadBalancerTarget.State.REGISTERED);
        assertThat(update.getPriority()).isEqualTo(Priority.LOW);
        assertThat(update.getLoadBalancerId()).isEqualTo(loadBalancerId);
    });
}
Also used : Task(com.netflix.titus.api.jobmanager.model.job.Task) JobLoadBalancerState(com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState) JobManagerException(com.netflix.titus.api.jobmanager.service.JobManagerException) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) Test(org.junit.Test)

Example 34 with JobLoadBalancer

use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.

the class DefaultLoadBalancerReconcilerTest method dissociatedJobsAreNotRemovedUntilAllTargetsAreDeregisteredAndRemoved.

@Test
public void dissociatedJobsAreNotRemovedUntilAllTargetsAreDeregisteredAndRemoved() throws InterruptedException {
    JobLoadBalancer jobLoadBalancer = new JobLoadBalancer(jobId, loadBalancerId);
    when(v3JobOperations.getTasks(jobId)).thenThrow(JobManagerException.jobNotFound(jobId));
    when(v3JobOperations.getJob(jobId)).thenReturn(Optional.empty());
    reset(connector);
    when(connector.getLoadBalancer(loadBalancerId)).thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.ACTIVE, Collections.singleton("1.2.3.4"))));
    store.addOrUpdateTargets(new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, "some-task", "1.2.3.4"), LoadBalancerTarget.State.DEREGISTERED)).block();
    assertThat(store.addOrUpdateLoadBalancer(jobLoadBalancer, State.DISSOCIATED).await(5, TimeUnit.SECONDS)).isTrue();
    testScheduler.triggerActions();
    subscriber.assertNotCompleted().assertNoValues();
    // 1. deregister
    awaitReconciliationRuns(1);
    subscriber.assertNoTerminalEvent().assertValueCount(1);
    assertThat(store.getAssociations()).isNotEmpty().hasSize(1);
    when(connector.getLoadBalancer(loadBalancerId)).thenReturn(Single.just(new LoadBalancer(loadBalancerId, LoadBalancer.State.ACTIVE, Collections.emptySet())));
    // Let a few iterations run so the remaining phases have a chance to complete:
    // 2. clean up target state
    // 3. clean up association
    awaitReconciliationRuns(3);
    assertThat(store.getAssociations()).isEmpty();
    assertThat(store.getLoadBalancerTargets(loadBalancerId).collectList().block()).isEmpty();
}
Also used : JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancer(com.netflix.titus.api.connector.cloud.LoadBalancer) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) Test(org.junit.Test)

Example 35 with JobLoadBalancer

use of com.netflix.titus.api.loadbalancer.model.JobLoadBalancer in project titus-control-plane by Netflix.

the class DefaultLoadBalancerServiceTest method backfillsCurrentTargetsToStore.

@Test(timeout = 30_000)
public void backfillsCurrentTargetsToStore() {
    String jobId = UUID.randomUUID().toString();
    String associatedId = "lb-" + UUID.randomUUID().toString();
    String dissociatedId = "lb-" + UUID.randomUUID().toString();
    String removedId = "lb-" + UUID.randomUUID().toString();
    LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
    when(configuration.isTargetsToStoreBackfillEnabled()).thenReturn(true);
    when(configuration.getStoreBackfillConcurrencyLimit()).thenReturn(10);
    when(configuration.getStoreBackfillTimeoutMs()).thenReturn(5000L);
    // current load balancer state (targets)
    when(client.getLoadBalancer(associatedId)).thenReturn(Single.just(new LoadBalancer(associatedId, LoadBalancer.State.ACTIVE, CollectionsExt.asSet("1.1.1.1", "2.2.2.2", "3.3.3.3"))));
    when(client.getLoadBalancer(dissociatedId)).thenReturn(Single.just(new LoadBalancer(dissociatedId, LoadBalancer.State.ACTIVE, CollectionsExt.asSet("4.4.4.4", "5.5.5.5", "6.6.6.6"))));
    when(client.getLoadBalancer(removedId)).thenReturn(Single.just(new LoadBalancer(removedId, LoadBalancer.State.REMOVED, Collections.emptySet())));
    // current load balancers we are managing
    loadBalancerStore.addOrUpdateLoadBalancer(new JobLoadBalancer(jobId, associatedId), JobLoadBalancer.State.ASSOCIATED).await();
    loadBalancerStore.addOrUpdateLoadBalancer(new JobLoadBalancer(jobId, dissociatedId), JobLoadBalancer.State.DISSOCIATED).await();
    loadBalancerStore.addOrUpdateLoadBalancer(new JobLoadBalancer(jobId, removedId), JobLoadBalancer.State.ASSOCIATED).await();
    DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
    service.backfillTargetsToStore();
    assertThat(loadBalancerStore.getLoadBalancerTargets(associatedId).collectList().block()).containsExactlyInAnyOrder(new LoadBalancerTargetState(new LoadBalancerTarget(associatedId, "BACKFILLED", "1.1.1.1"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(associatedId, "BACKFILLED", "2.2.2.2"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(associatedId, "BACKFILLED", "3.3.3.3"), REGISTERED));
    assertThat(loadBalancerStore.getLoadBalancerTargets(dissociatedId).collectList().block()).containsExactlyInAnyOrder(new LoadBalancerTargetState(new LoadBalancerTarget(dissociatedId, "BACKFILLED", "4.4.4.4"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(dissociatedId, "BACKFILLED", "5.5.5.5"), REGISTERED), new LoadBalancerTargetState(new LoadBalancerTarget(dissociatedId, "BACKFILLED", "6.6.6.6"), REGISTERED));
    assertThat(loadBalancerStore.getLoadBalancerTargets(removedId).collectList().block()).isEmpty();
}
Also used : JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancer(com.netflix.titus.api.connector.cloud.LoadBalancer) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) Test(org.junit.Test)

Aggregations

JobLoadBalancer (com.netflix.titus.api.loadbalancer.model.JobLoadBalancer)36 Test (org.junit.Test)27 JobLoadBalancerState (com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState)14 Task (com.netflix.titus.api.jobmanager.model.job.Task)10 LoadBalancerTarget (com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget)10 LoadBalancerTargetState (com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState)10 LoadBalancer (com.netflix.titus.api.connector.cloud.LoadBalancer)7 List (java.util.List)6 GetJobLoadBalancersResult (com.netflix.titus.grpc.protogen.GetJobLoadBalancersResult)5 Collections (java.util.Collections)5 Collectors (java.util.stream.Collectors)5 JobManagerException (com.netflix.titus.api.jobmanager.service.JobManagerException)4 LoadBalancerStore (com.netflix.titus.api.loadbalancer.store.LoadBalancerStore)4 GetAllLoadBalancersResult (com.netflix.titus.grpc.protogen.GetAllLoadBalancersResult)4 Completable (rx.Completable)4 Schedulers (rx.schedulers.Schedulers)4 LoadBalancerConnector (com.netflix.titus.api.connector.cloud.LoadBalancerConnector)3 TaskAttributes (com.netflix.titus.api.jobmanager.TaskAttributes)3 V3JobOperations (com.netflix.titus.api.jobmanager.service.V3JobOperations)3 ArrayList (java.util.ArrayList)3