Search in sources :

Example 11 with V3JobOperations

use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.

the class DefaultLoadBalancerServiceTest method movedTasks.

@Test
public void movedTasks() {
    String taskId = UUID.randomUUID().toString();
    String sourceJobId = UUID.randomUUID().toString();
    String targetJobId = UUID.randomUUID().toString();
    String sourceLoadBalancerId = "lb-" + UUID.randomUUID().toString();
    String targetLoadBalancerId = "lb-" + UUID.randomUUID().toString();
    String commonLoadBalancerId = "lb-" + UUID.randomUUID().toString();
    PublishSubject<JobManagerEvent<?>> taskEvents = PublishSubject.create();
    when(client.registerAll(any(), any())).thenReturn(Completable.complete());
    when(client.deregisterAll(any(), any())).thenReturn(Completable.complete());
    when(v3JobOperations.observeJobs()).thenReturn(taskEvents);
    LoadBalancerTests.applyValidGetJobMock(v3JobOperations, sourceJobId);
    LoadBalancerTests.applyValidGetJobMock(v3JobOperations, targetJobId);
    LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
    DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
    AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
    assertTrue(service.addLoadBalancer(sourceJobId, sourceLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
    assertTrue(service.addLoadBalancer(sourceJobId, commonLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
    assertThat(service.getJobLoadBalancers(sourceJobId).toBlocking().toIterable()).containsExactlyInAnyOrder(sourceLoadBalancerId, commonLoadBalancerId);
    assertTrue(service.addLoadBalancer(targetJobId, targetLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
    assertTrue(service.addLoadBalancer(targetJobId, commonLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
    assertThat(service.getJobLoadBalancers(targetJobId).toBlocking().toIterable()).containsExactlyInAnyOrder(targetLoadBalancerId, commonLoadBalancerId);
    testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
    testSubscriber.assertNoErrors().assertValueCount(0);
    verify(client, never()).registerAll(any(), any());
    verify(client, never()).deregisterAll(any(), any());
    verifyNoReconcilerIgnore();
    Task moved = ServiceJobTask.newBuilder().withJobId(targetJobId).withId(taskId).withStatus(TaskStatus.newBuilder().withState(TaskState.Started).build()).withTaskContext(CollectionsExt.asMap(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "1.2.3.4", TaskAttributes.TASK_ATTRIBUTES_MOVED_FROM_JOB, sourceJobId)).build();
    // detect the task is moved, gets deregistered from the source and registered on the target
    taskEvents.onNext(TaskUpdateEvent.newTaskFromAnotherJob(null, moved, callMetadata));
    testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
    testSubscriber.assertNoErrors().assertValueCount(2);
    verify(client).registerAll(eq(targetLoadBalancerId), argThat(set -> set.contains("1.2.3.4")));
    verify(client).deregisterAll(eq(sourceLoadBalancerId), argThat(set -> set.contains("1.2.3.4")));
    verifyReconcilerIgnore(targetLoadBalancerId, "1.2.3.4");
    verifyReconcilerIgnore(sourceLoadBalancerId, "1.2.3.4");
    // load balancers associated with both source and target jobs are not changed
    verify(client, never()).registerAll(eq(commonLoadBalancerId), any());
    verify(client, never()).deregisterAll(eq(commonLoadBalancerId), any());
    verifyNoReconcilerIgnore(commonLoadBalancerId);
    verifyNoReconcilerIgnore(commonLoadBalancerId);
}
Also used : LoadBalancerConnector(com.netflix.titus.api.connector.cloud.LoadBalancerConnector) JobLoadBalancerState(com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState) ArgumentMatchers.argThat(org.mockito.ArgumentMatchers.argThat) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Task(com.netflix.titus.api.jobmanager.model.job.Task) CollectionsExt(com.netflix.titus.common.util.CollectionsExt) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Single(rx.Single) TitusRuntimes(com.netflix.titus.common.runtime.TitusRuntimes) DefaultLoadBalancerJobValidator(com.netflix.titus.api.loadbalancer.model.sanitizer.DefaultLoadBalancerJobValidator) Schedulers(rx.schedulers.Schedulers) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) LoadBalancerStore(com.netflix.titus.api.loadbalancer.store.LoadBalancerStore) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancerValidationConfiguration(com.netflix.titus.api.loadbalancer.model.sanitizer.LoadBalancerValidationConfiguration) Collection(java.util.Collection) DEREGISTERED(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget.State.DEREGISTERED) TaskStatus(com.netflix.titus.api.jobmanager.model.job.TaskStatus) Set(java.util.Set) UUID(java.util.UUID) JobManagerEvent(com.netflix.titus.api.jobmanager.model.job.event.JobManagerEvent) Collectors(java.util.stream.Collectors) TaskState(com.netflix.titus.api.jobmanager.model.job.TaskState) List(java.util.List) TaskUpdateEvent(com.netflix.titus.api.jobmanager.model.job.event.TaskUpdateEvent) Assert.assertFalse(org.junit.Assert.assertFalse) PublishSubject(rx.subjects.PublishSubject) Mockito.mock(org.mockito.Mockito.mock) Completable(rx.Completable) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) LoadBalancerJobValidator(com.netflix.titus.api.loadbalancer.model.sanitizer.LoadBalancerJobValidator) TestScheduler(rx.schedulers.TestScheduler) ArrayList(java.util.ArrayList) AssertableSubscriber(rx.observers.AssertableSubscriber) ServiceJobTask(com.netflix.titus.api.jobmanager.model.job.ServiceJobTask) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) JobManagerException(com.netflix.titus.api.jobmanager.service.JobManagerException) Before(org.junit.Before) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) LoadBalancer(com.netflix.titus.api.connector.cloud.LoadBalancer) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) Batch(com.netflix.titus.common.util.rx.batch.Batch) Mockito.never(org.mockito.Mockito.never) TaskAttributes(com.netflix.titus.api.jobmanager.TaskAttributes) InMemoryLoadBalancerStore(com.netflix.titus.runtime.store.v3.memory.InMemoryLoadBalancerStore) REGISTERED(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget.State.REGISTERED) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Task(com.netflix.titus.api.jobmanager.model.job.Task) ServiceJobTask(com.netflix.titus.api.jobmanager.model.job.ServiceJobTask) Batch(com.netflix.titus.common.util.rx.batch.Batch) JobManagerEvent(com.netflix.titus.api.jobmanager.model.job.event.JobManagerEvent) Test(org.junit.Test)

Example 12 with V3JobOperations

use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.

the class DefaultLoadBalancerServiceTest method addLoadBalancerRegistersTasks.

@Test
public void addLoadBalancerRegistersTasks() {
    String jobId = UUID.randomUUID().toString();
    String loadBalancerId = "lb-" + UUID.randomUUID().toString();
    defaultStubs();
    LoadBalancerTests.applyValidGetJobMock(v3JobOperations, jobId);
    List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
    Collection<LoadBalancerTargetState> expectedTargets = tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), REGISTERED)).collect(Collectors.toList());
    when(v3JobOperations.getTasks(jobId)).thenReturn(CollectionsExt.merge(tasks, LoadBalancerTests.buildTasks(2, jobId, TaskState.StartInitiated), LoadBalancerTests.buildTasks(2, jobId, TaskState.KillInitiated), LoadBalancerTests.buildTasks(3, jobId, TaskState.Finished), LoadBalancerTests.buildTasks(1, jobId, TaskState.Disconnected)));
    LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
    DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
    AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
    assertTrue(service.addLoadBalancer(jobId, loadBalancerId).await(100, TimeUnit.MILLISECONDS));
    assertThat(service.getJobLoadBalancers(jobId).toBlocking().first()).isEqualTo(loadBalancerId);
    verify(v3JobOperations).getTasks(jobId);
    testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
    testSubscriber.assertNoErrors().assertValueCount(1);
    assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).containsExactlyInAnyOrderElementsOf(expectedTargets);
    verify(client).registerAll(eq(loadBalancerId), argThat(targets -> targets != null && targets.size() == tasks.size()));
    verify(client, never()).deregisterAll(eq(loadBalancerId), any());
    verifyReconcilerIgnore(loadBalancerId, LoadBalancerTests.ipAddresses(tasks));
}
Also used : LoadBalancerConnector(com.netflix.titus.api.connector.cloud.LoadBalancerConnector) JobLoadBalancerState(com.netflix.titus.api.loadbalancer.model.JobLoadBalancerState) ArgumentMatchers.argThat(org.mockito.ArgumentMatchers.argThat) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Task(com.netflix.titus.api.jobmanager.model.job.Task) CollectionsExt(com.netflix.titus.common.util.CollectionsExt) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Single(rx.Single) TitusRuntimes(com.netflix.titus.common.runtime.TitusRuntimes) DefaultLoadBalancerJobValidator(com.netflix.titus.api.loadbalancer.model.sanitizer.DefaultLoadBalancerJobValidator) Schedulers(rx.schedulers.Schedulers) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) LoadBalancerStore(com.netflix.titus.api.loadbalancer.store.LoadBalancerStore) JobLoadBalancer(com.netflix.titus.api.loadbalancer.model.JobLoadBalancer) LoadBalancerValidationConfiguration(com.netflix.titus.api.loadbalancer.model.sanitizer.LoadBalancerValidationConfiguration) Collection(java.util.Collection) DEREGISTERED(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget.State.DEREGISTERED) TaskStatus(com.netflix.titus.api.jobmanager.model.job.TaskStatus) Set(java.util.Set) UUID(java.util.UUID) JobManagerEvent(com.netflix.titus.api.jobmanager.model.job.event.JobManagerEvent) Collectors(java.util.stream.Collectors) TaskState(com.netflix.titus.api.jobmanager.model.job.TaskState) List(java.util.List) TaskUpdateEvent(com.netflix.titus.api.jobmanager.model.job.event.TaskUpdateEvent) Assert.assertFalse(org.junit.Assert.assertFalse) PublishSubject(rx.subjects.PublishSubject) Mockito.mock(org.mockito.Mockito.mock) Completable(rx.Completable) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) LoadBalancerJobValidator(com.netflix.titus.api.loadbalancer.model.sanitizer.LoadBalancerJobValidator) TestScheduler(rx.schedulers.TestScheduler) ArrayList(java.util.ArrayList) AssertableSubscriber(rx.observers.AssertableSubscriber) ServiceJobTask(com.netflix.titus.api.jobmanager.model.job.ServiceJobTask) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) JobManagerException(com.netflix.titus.api.jobmanager.service.JobManagerException) Before(org.junit.Before) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) LoadBalancer(com.netflix.titus.api.connector.cloud.LoadBalancer) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) Batch(com.netflix.titus.common.util.rx.batch.Batch) Mockito.never(org.mockito.Mockito.never) TaskAttributes(com.netflix.titus.api.jobmanager.TaskAttributes) InMemoryLoadBalancerStore(com.netflix.titus.runtime.store.v3.memory.InMemoryLoadBalancerStore) REGISTERED(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget.State.REGISTERED) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Task(com.netflix.titus.api.jobmanager.model.job.Task) ServiceJobTask(com.netflix.titus.api.jobmanager.model.job.ServiceJobTask) Batch(com.netflix.titus.common.util.rx.batch.Batch) LoadBalancerTargetState(com.netflix.titus.api.loadbalancer.model.LoadBalancerTargetState) LoadBalancerTarget(com.netflix.titus.api.loadbalancer.model.LoadBalancerTarget) Test(org.junit.Test)

Example 13 with V3JobOperations

use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluatorTest method batchJobWithMultipleTasks.

@SuppressWarnings("unchecked")
@Test
public void batchJobWithMultipleTasks() {
    when(applicationSlaManagementService.getApplicationSLAs()).thenReturn(asList(ConsumptionModelGenerator.DEFAULT_SLA, ConsumptionModelGenerator.CRITICAL_SLA_1, ConsumptionModelGenerator.NOT_USED_SLA));
    // Job with defined capacity group SLA
    Job<BatchJobExt> goodCapacityJob = newBatchJob("goodCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).build()).getLeft();
    List<Task> goodCapacityTasks = jobComponentStub.getJobOperations().getTasks(goodCapacityJob.getId());
    // Job without appName defined
    Job<BatchJobExt> noAppNameJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withApplicationName("").withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).build()).getLeft();
    List<Task> noAppNameTasks = jobComponentStub.getJobOperations().getTasks(noAppNameJob.getId());
    // Job with capacity group for which SLA is not defined
    Job<BatchJobExt> badCapacityJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup("missingCapacityGroup").build()).getLeft();
    List<Task> badCapacityTasks = jobComponentStub.getJobOperations().getTasks(badCapacityJob.getId());
    // Evaluate
    ResourceConsumptionEvaluator evaluator = new ResourceConsumptionEvaluator(applicationSlaManagementService, v3JobOperations);
    Set<String> undefined = evaluator.getUndefinedCapacityGroups();
    assertThat(undefined).contains("missingCapacityGroup");
    CompositeResourceConsumption systemConsumption = evaluator.getSystemConsumption();
    Map<String, ResourceConsumption> tierConsumptions = systemConsumption.getContributors();
    assertThat(tierConsumptions).containsKeys(Tier.Critical.name(), Tier.Flex.name());
    // Critical capacity group
    CompositeResourceConsumption criticalConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).get();
    assertThat(criticalConsumption.getCurrentConsumption()).isEqualTo(expectedCurrentConsumptionForBatchJob(goodCapacityJob, goodCapacityTasks));
    assertThat(criticalConsumption.getMaxConsumption()).isEqualTo(expectedMaxConsumptionForBatchJob(goodCapacityJob));
    assertThat(criticalConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.CRITICAL_SLA_1));
    assertThat(criticalConsumption.isAboveLimit()).isTrue();
    // Default capacity group
    CompositeResourceConsumption defaultConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Flex.name(), ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).get();
    assertThat(defaultConsumption.getCurrentConsumption()).isEqualTo(ResourceDimensions.add(expectedCurrentConsumptionForBatchJob(noAppNameJob, noAppNameTasks), expectedCurrentConsumptionForBatchJob(badCapacityJob, badCapacityTasks)));
    assertThat(defaultConsumption.getMaxConsumption()).isEqualTo(ResourceDimensions.add(expectedMaxConsumptionForBatchJob(noAppNameJob), expectedMaxConsumptionForBatchJob(badCapacityJob)));
    assertThat(defaultConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.DEFAULT_SLA));
    assertThat(defaultConsumption.isAboveLimit()).isFalse();
    // Not used capacity group
    CompositeResourceConsumption notUsedConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.NOT_USED_SLA.getAppName()).get();
    assertThat(notUsedConsumption.getCurrentConsumption()).isEqualTo(ResourceDimension.empty());
    assertThat(notUsedConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.NOT_USED_SLA));
    assertThat(notUsedConsumption.isAboveLimit()).isFalse();
}
Also used : JobDescriptorGenerator(com.netflix.titus.testkit.model.job.JobDescriptorGenerator) ApplicationSlaManagementService(com.netflix.titus.master.service.management.ApplicationSlaManagementService) Task(com.netflix.titus.api.jobmanager.model.job.Task) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) ResourceDimensions(com.netflix.titus.master.model.ResourceDimensions) Function(java.util.function.Function) ResourceConsumption(com.netflix.titus.master.service.management.ResourceConsumption) Pair(com.netflix.titus.common.util.tuple.Pair) TitusRuntimes(com.netflix.titus.common.runtime.TitusRuntimes) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) BatchJobExt(com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt) ResourceDimension(com.netflix.titus.api.model.ResourceDimension) ResourceConsumptionEvaluator.perTaskResourceDimension(com.netflix.titus.master.service.management.internal.ResourceConsumptionEvaluator.perTaskResourceDimension) Before(org.junit.Before) JobDescriptor(com.netflix.titus.api.jobmanager.model.job.JobDescriptor) Job(com.netflix.titus.api.jobmanager.model.job.Job) Set(java.util.Set) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Tier(com.netflix.titus.api.model.Tier) ResourceConsumptions.findConsumption(com.netflix.titus.master.service.management.ResourceConsumptions.findConsumption) TaskState(com.netflix.titus.api.jobmanager.model.job.TaskState) List(java.util.List) V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) ContainerResources(com.netflix.titus.api.jobmanager.model.job.ContainerResources) JobComponentStub(com.netflix.titus.testkit.model.job.JobComponentStub) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) CompositeResourceConsumption(com.netflix.titus.master.service.management.CompositeResourceConsumption) Mockito.mock(org.mockito.Mockito.mock) Task(com.netflix.titus.api.jobmanager.model.job.Task) CompositeResourceConsumption(com.netflix.titus.master.service.management.CompositeResourceConsumption) ResourceConsumption(com.netflix.titus.master.service.management.ResourceConsumption) CompositeResourceConsumption(com.netflix.titus.master.service.management.CompositeResourceConsumption) BatchJobExt(com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt) Test(org.junit.Test)

Example 14 with V3JobOperations

use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.

the class DefaultAppScaleManagerTest method checkBulkPolicyCreations.

@Test
public void checkBulkPolicyCreations() {
    AutoScalingPolicyTests.MockAlarmClient mockAlarmClient = new AutoScalingPolicyTests.MockAlarmClient();
    AutoScalingPolicyTests.MockAppAutoScalingClient mockAppAutoScalingClient = new AutoScalingPolicyTests.MockAppAutoScalingClient();
    InMemoryPolicyStore policyStore = new InMemoryPolicyStore();
    final int totalJobs = 300;
    List<String> jobIds = new ArrayList<>(totalJobs);
    for (int i = 0; i < totalJobs; i++) {
        String jobId = UUID.randomUUID().toString();
        jobIds.add(jobId);
    }
    V3JobOperations v3JobOperations = mockV3OperationsForJobs(jobIds);
    DefaultAppScaleManager appScaleManager = new DefaultAppScaleManager(policyStore, mockAlarmClient, mockAppAutoScalingClient, v3JobOperations, new DefaultRegistry(), AutoScalingPolicyTests.mockAppScaleManagerConfiguration(), Schedulers.computation(), mock(TitusRuntime.class));
    final CountDownLatch latch = new CountDownLatch(totalJobs);
    for (int i = 0; i < totalJobs; i++) {
        final AutoScalingPolicy autoScalingPolicy = AutoScalingPolicyTests.buildStepScalingPolicy(jobIds.get(i));
        appScaleManager.createAutoScalingPolicy(autoScalingPolicy).subscribe(pid -> {
            log.info("Policy created {}", pid);
            latch.countDown();
        });
    }
    await().timeout(30, TimeUnit.SECONDS).until(() -> mockAppAutoScalingClient.getNumPolicies() == totalJobs);
}
Also used : V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) ArrayList(java.util.ArrayList) InMemoryPolicyStore(com.netflix.titus.runtime.store.v3.memory.InMemoryPolicyStore) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) CountDownLatch(java.util.concurrent.CountDownLatch) AutoScalingPolicy(com.netflix.titus.api.appscale.model.AutoScalingPolicy) DefaultRegistry(com.netflix.spectator.api.DefaultRegistry) Test(org.junit.Test)

Example 15 with V3JobOperations

use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.

the class DefaultAppScaleManagerTest method checkCreatePolicyFlow.

private void checkCreatePolicyFlow(PolicyType policyType) throws Exception {
    // create instance of DefaultAppScaleManager
    AutoScalingPolicyTests.MockAlarmClient mockAlarmClient = new AutoScalingPolicyTests.MockAlarmClient();
    AutoScalingPolicyTests.MockAppAutoScalingClient mockAppAutoScalingClient = new AutoScalingPolicyTests.MockAppAutoScalingClient();
    InMemoryPolicyStore policyStore = new InMemoryPolicyStore();
    String jobIdOne = UUID.randomUUID().toString();
    String jobIdTwo = UUID.randomUUID().toString();
    V3JobOperations v3JobOperations = mockV3Operations(jobIdOne, jobIdTwo);
    DefaultAppScaleManager appScaleManager = new DefaultAppScaleManager(policyStore, mockAlarmClient, mockAppAutoScalingClient, v3JobOperations, new DefaultRegistry(), AutoScalingPolicyTests.mockAppScaleManagerConfiguration(), Schedulers.immediate(), mock(TitusRuntime.class));
    AutoScalingPolicy autoScalingPolicyOne;
    AutoScalingPolicy autoScalingPolicyTwo;
    if (policyType == PolicyType.StepScaling) {
        autoScalingPolicyOne = AutoScalingPolicyTests.buildStepScalingPolicy(jobIdOne);
        autoScalingPolicyTwo = AutoScalingPolicyTests.buildStepScalingPolicy(jobIdTwo);
    } else {
        autoScalingPolicyOne = AutoScalingPolicyTests.buildTargetTrackingPolicy(jobIdOne);
        autoScalingPolicyTwo = AutoScalingPolicyTests.buildTargetTrackingPolicy(jobIdTwo);
    }
    // call - createAutoScalingPolicy
    appScaleManager.createAutoScalingPolicy(autoScalingPolicyOne).toBlocking().single();
    String policyRefIdTwo = appScaleManager.createAutoScalingPolicy(autoScalingPolicyTwo).toBlocking().single();
    AutoScalingPolicyTests.waitForCondition(() -> {
        List<AutoScalingPolicy> policies = policyStore.retrievePolicies(false).toList().toBlocking().first();
        return policies.size() == 2 && mockAppAutoScalingClient.getNumPolicies() == 2 && mockAppAutoScalingClient.getNumScalableTargets() == 2;
    });
    // verify counts in CloudAlarmClient, AppAutoScaleClient and AppScalePolicyStore
    List<AutoScalingPolicy> policiesStored = policyStore.retrievePolicies(false).toList().toBlocking().first();
    Assertions.assertThat(policiesStored.size()).isEqualTo(2);
    Assertions.assertThat(mockAppAutoScalingClient.getNumPolicies()).isEqualTo(2);
    Assertions.assertThat(mockAppAutoScalingClient.getNumScalableTargets()).isEqualTo(2);
    if (policyType == PolicyType.StepScaling) {
        Assertions.assertThat(mockAlarmClient.getNumOfAlarmsCreated()).isEqualTo(2);
    }
    appScaleManager.removeAutoScalingPolicy(policyRefIdTwo).await();
    AutoScalingPolicyTests.waitForCondition(() -> {
        List<AutoScalingPolicy> policies = policyStore.retrievePolicies(false).toList().toBlocking().first();
        return policies.size() == 1 && mockAppAutoScalingClient.getNumPolicies() == 1 && mockAppAutoScalingClient.getNumScalableTargets() == 1;
    });
    // verify counts in CloudAlarmClient, AppAutoScaleClient and AppScalePolicyStore
    policiesStored = policyStore.retrievePolicies(false).toList().toBlocking().first();
    Assertions.assertThat(policiesStored.size()).isEqualTo(1);
    Assertions.assertThat(mockAppAutoScalingClient.getNumPolicies()).isEqualTo(1);
    Assertions.assertThat(mockAppAutoScalingClient.getNumScalableTargets()).isEqualTo(1);
    if (policyType == PolicyType.StepScaling) {
        Assertions.assertThat(mockAlarmClient.getNumOfAlarmsCreated()).isEqualTo(1);
    }
}
Also used : V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) InMemoryPolicyStore(com.netflix.titus.runtime.store.v3.memory.InMemoryPolicyStore) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) AutoScalingPolicy(com.netflix.titus.api.appscale.model.AutoScalingPolicy) DefaultRegistry(com.netflix.spectator.api.DefaultRegistry)

Aggregations

V3JobOperations (com.netflix.titus.api.jobmanager.service.V3JobOperations)20 TitusRuntime (com.netflix.titus.common.runtime.TitusRuntime)16 LoadBalancerConnector (com.netflix.titus.api.connector.cloud.LoadBalancerConnector)14 InMemoryLoadBalancerStore (com.netflix.titus.runtime.store.v3.memory.InMemoryLoadBalancerStore)14 Test (org.junit.Test)14 ArrayList (java.util.ArrayList)13 LoadBalancer (com.netflix.titus.api.connector.cloud.LoadBalancer)12 Task (com.netflix.titus.api.jobmanager.model.job.Task)12 TaskState (com.netflix.titus.api.jobmanager.model.job.TaskState)12 JobLoadBalancer (com.netflix.titus.api.loadbalancer.model.JobLoadBalancer)12 DefaultLoadBalancerJobValidator (com.netflix.titus.api.loadbalancer.model.sanitizer.DefaultLoadBalancerJobValidator)12 LoadBalancerJobValidator (com.netflix.titus.api.loadbalancer.model.sanitizer.LoadBalancerJobValidator)12 LoadBalancerValidationConfiguration (com.netflix.titus.api.loadbalancer.model.sanitizer.LoadBalancerValidationConfiguration)12 LoadBalancerStore (com.netflix.titus.api.loadbalancer.store.LoadBalancerStore)12 TitusRuntimes (com.netflix.titus.common.runtime.TitusRuntimes)12 List (java.util.List)12 Set (java.util.Set)12 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)12 TaskAttributes (com.netflix.titus.api.jobmanager.TaskAttributes)11 ServiceJobTask (com.netflix.titus.api.jobmanager.model.job.ServiceJobTask)11