use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.
the class DefaultLoadBalancerServiceTest method movedTasks.
@Test
public void movedTasks() {
String taskId = UUID.randomUUID().toString();
String sourceJobId = UUID.randomUUID().toString();
String targetJobId = UUID.randomUUID().toString();
String sourceLoadBalancerId = "lb-" + UUID.randomUUID().toString();
String targetLoadBalancerId = "lb-" + UUID.randomUUID().toString();
String commonLoadBalancerId = "lb-" + UUID.randomUUID().toString();
PublishSubject<JobManagerEvent<?>> taskEvents = PublishSubject.create();
when(client.registerAll(any(), any())).thenReturn(Completable.complete());
when(client.deregisterAll(any(), any())).thenReturn(Completable.complete());
when(v3JobOperations.observeJobs()).thenReturn(taskEvents);
LoadBalancerTests.applyValidGetJobMock(v3JobOperations, sourceJobId);
LoadBalancerTests.applyValidGetJobMock(v3JobOperations, targetJobId);
LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
assertTrue(service.addLoadBalancer(sourceJobId, sourceLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertTrue(service.addLoadBalancer(sourceJobId, commonLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertThat(service.getJobLoadBalancers(sourceJobId).toBlocking().toIterable()).containsExactlyInAnyOrder(sourceLoadBalancerId, commonLoadBalancerId);
assertTrue(service.addLoadBalancer(targetJobId, targetLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertTrue(service.addLoadBalancer(targetJobId, commonLoadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertThat(service.getJobLoadBalancers(targetJobId).toBlocking().toIterable()).containsExactlyInAnyOrder(targetLoadBalancerId, commonLoadBalancerId);
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(0);
verify(client, never()).registerAll(any(), any());
verify(client, never()).deregisterAll(any(), any());
verifyNoReconcilerIgnore();
Task moved = ServiceJobTask.newBuilder().withJobId(targetJobId).withId(taskId).withStatus(TaskStatus.newBuilder().withState(TaskState.Started).build()).withTaskContext(CollectionsExt.asMap(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP, "1.2.3.4", TaskAttributes.TASK_ATTRIBUTES_MOVED_FROM_JOB, sourceJobId)).build();
// detect the task is moved, gets deregistered from the source and registered on the target
taskEvents.onNext(TaskUpdateEvent.newTaskFromAnotherJob(null, moved, callMetadata));
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(2);
verify(client).registerAll(eq(targetLoadBalancerId), argThat(set -> set.contains("1.2.3.4")));
verify(client).deregisterAll(eq(sourceLoadBalancerId), argThat(set -> set.contains("1.2.3.4")));
verifyReconcilerIgnore(targetLoadBalancerId, "1.2.3.4");
verifyReconcilerIgnore(sourceLoadBalancerId, "1.2.3.4");
// load balancers associated with both source and target jobs are not changed
verify(client, never()).registerAll(eq(commonLoadBalancerId), any());
verify(client, never()).deregisterAll(eq(commonLoadBalancerId), any());
verifyNoReconcilerIgnore(commonLoadBalancerId);
verifyNoReconcilerIgnore(commonLoadBalancerId);
}
use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.
the class DefaultLoadBalancerServiceTest method addLoadBalancerRegistersTasks.
@Test
public void addLoadBalancerRegistersTasks() {
String jobId = UUID.randomUUID().toString();
String loadBalancerId = "lb-" + UUID.randomUUID().toString();
defaultStubs();
LoadBalancerTests.applyValidGetJobMock(v3JobOperations, jobId);
List<Task> tasks = LoadBalancerTests.buildTasksStarted(5, jobId);
Collection<LoadBalancerTargetState> expectedTargets = tasks.stream().map(task -> new LoadBalancerTargetState(new LoadBalancerTarget(loadBalancerId, task.getId(), task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_CONTAINER_IP)), REGISTERED)).collect(Collectors.toList());
when(v3JobOperations.getTasks(jobId)).thenReturn(CollectionsExt.merge(tasks, LoadBalancerTests.buildTasks(2, jobId, TaskState.StartInitiated), LoadBalancerTests.buildTasks(2, jobId, TaskState.KillInitiated), LoadBalancerTests.buildTasks(3, jobId, TaskState.Finished), LoadBalancerTests.buildTasks(1, jobId, TaskState.Disconnected)));
LoadBalancerConfiguration configuration = LoadBalancerTests.mockConfiguration(MIN_TIME_IN_QUEUE_MS);
DefaultLoadBalancerService service = new DefaultLoadBalancerService(runtime, configuration, client, loadBalancerStore, loadBalancerJobOperations, reconciler, validator, testScheduler);
AssertableSubscriber<Batch<TargetStateBatchable, String>> testSubscriber = service.events().test();
assertTrue(service.addLoadBalancer(jobId, loadBalancerId).await(100, TimeUnit.MILLISECONDS));
assertThat(service.getJobLoadBalancers(jobId).toBlocking().first()).isEqualTo(loadBalancerId);
verify(v3JobOperations).getTasks(jobId);
testScheduler.advanceTimeBy(FLUSH_WAIT_TIME_MS, TimeUnit.MILLISECONDS);
testSubscriber.assertNoErrors().assertValueCount(1);
assertThat(loadBalancerStore.getLoadBalancerTargets(loadBalancerId).collectList().block()).containsExactlyInAnyOrderElementsOf(expectedTargets);
verify(client).registerAll(eq(loadBalancerId), argThat(targets -> targets != null && targets.size() == tasks.size()));
verify(client, never()).deregisterAll(eq(loadBalancerId), any());
verifyReconcilerIgnore(loadBalancerId, LoadBalancerTests.ipAddresses(tasks));
}
use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.
the class ResourceConsumptionEvaluatorTest method batchJobWithMultipleTasks.
@SuppressWarnings("unchecked")
@Test
public void batchJobWithMultipleTasks() {
when(applicationSlaManagementService.getApplicationSLAs()).thenReturn(asList(ConsumptionModelGenerator.DEFAULT_SLA, ConsumptionModelGenerator.CRITICAL_SLA_1, ConsumptionModelGenerator.NOT_USED_SLA));
// Job with defined capacity group SLA
Job<BatchJobExt> goodCapacityJob = newBatchJob("goodCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).build()).getLeft();
List<Task> goodCapacityTasks = jobComponentStub.getJobOperations().getTasks(goodCapacityJob.getId());
// Job without appName defined
Job<BatchJobExt> noAppNameJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withApplicationName("").withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).build()).getLeft();
List<Task> noAppNameTasks = jobComponentStub.getJobOperations().getTasks(noAppNameJob.getId());
// Job with capacity group for which SLA is not defined
Job<BatchJobExt> badCapacityJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup("missingCapacityGroup").build()).getLeft();
List<Task> badCapacityTasks = jobComponentStub.getJobOperations().getTasks(badCapacityJob.getId());
// Evaluate
ResourceConsumptionEvaluator evaluator = new ResourceConsumptionEvaluator(applicationSlaManagementService, v3JobOperations);
Set<String> undefined = evaluator.getUndefinedCapacityGroups();
assertThat(undefined).contains("missingCapacityGroup");
CompositeResourceConsumption systemConsumption = evaluator.getSystemConsumption();
Map<String, ResourceConsumption> tierConsumptions = systemConsumption.getContributors();
assertThat(tierConsumptions).containsKeys(Tier.Critical.name(), Tier.Flex.name());
// Critical capacity group
CompositeResourceConsumption criticalConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).get();
assertThat(criticalConsumption.getCurrentConsumption()).isEqualTo(expectedCurrentConsumptionForBatchJob(goodCapacityJob, goodCapacityTasks));
assertThat(criticalConsumption.getMaxConsumption()).isEqualTo(expectedMaxConsumptionForBatchJob(goodCapacityJob));
assertThat(criticalConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.CRITICAL_SLA_1));
assertThat(criticalConsumption.isAboveLimit()).isTrue();
// Default capacity group
CompositeResourceConsumption defaultConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Flex.name(), ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).get();
assertThat(defaultConsumption.getCurrentConsumption()).isEqualTo(ResourceDimensions.add(expectedCurrentConsumptionForBatchJob(noAppNameJob, noAppNameTasks), expectedCurrentConsumptionForBatchJob(badCapacityJob, badCapacityTasks)));
assertThat(defaultConsumption.getMaxConsumption()).isEqualTo(ResourceDimensions.add(expectedMaxConsumptionForBatchJob(noAppNameJob), expectedMaxConsumptionForBatchJob(badCapacityJob)));
assertThat(defaultConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.DEFAULT_SLA));
assertThat(defaultConsumption.isAboveLimit()).isFalse();
// Not used capacity group
CompositeResourceConsumption notUsedConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.NOT_USED_SLA.getAppName()).get();
assertThat(notUsedConsumption.getCurrentConsumption()).isEqualTo(ResourceDimension.empty());
assertThat(notUsedConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.NOT_USED_SLA));
assertThat(notUsedConsumption.isAboveLimit()).isFalse();
}
use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.
the class DefaultAppScaleManagerTest method checkBulkPolicyCreations.
@Test
public void checkBulkPolicyCreations() {
AutoScalingPolicyTests.MockAlarmClient mockAlarmClient = new AutoScalingPolicyTests.MockAlarmClient();
AutoScalingPolicyTests.MockAppAutoScalingClient mockAppAutoScalingClient = new AutoScalingPolicyTests.MockAppAutoScalingClient();
InMemoryPolicyStore policyStore = new InMemoryPolicyStore();
final int totalJobs = 300;
List<String> jobIds = new ArrayList<>(totalJobs);
for (int i = 0; i < totalJobs; i++) {
String jobId = UUID.randomUUID().toString();
jobIds.add(jobId);
}
V3JobOperations v3JobOperations = mockV3OperationsForJobs(jobIds);
DefaultAppScaleManager appScaleManager = new DefaultAppScaleManager(policyStore, mockAlarmClient, mockAppAutoScalingClient, v3JobOperations, new DefaultRegistry(), AutoScalingPolicyTests.mockAppScaleManagerConfiguration(), Schedulers.computation(), mock(TitusRuntime.class));
final CountDownLatch latch = new CountDownLatch(totalJobs);
for (int i = 0; i < totalJobs; i++) {
final AutoScalingPolicy autoScalingPolicy = AutoScalingPolicyTests.buildStepScalingPolicy(jobIds.get(i));
appScaleManager.createAutoScalingPolicy(autoScalingPolicy).subscribe(pid -> {
log.info("Policy created {}", pid);
latch.countDown();
});
}
await().timeout(30, TimeUnit.SECONDS).until(() -> mockAppAutoScalingClient.getNumPolicies() == totalJobs);
}
use of com.netflix.titus.api.jobmanager.service.V3JobOperations in project titus-control-plane by Netflix.
the class DefaultAppScaleManagerTest method checkCreatePolicyFlow.
private void checkCreatePolicyFlow(PolicyType policyType) throws Exception {
// create instance of DefaultAppScaleManager
AutoScalingPolicyTests.MockAlarmClient mockAlarmClient = new AutoScalingPolicyTests.MockAlarmClient();
AutoScalingPolicyTests.MockAppAutoScalingClient mockAppAutoScalingClient = new AutoScalingPolicyTests.MockAppAutoScalingClient();
InMemoryPolicyStore policyStore = new InMemoryPolicyStore();
String jobIdOne = UUID.randomUUID().toString();
String jobIdTwo = UUID.randomUUID().toString();
V3JobOperations v3JobOperations = mockV3Operations(jobIdOne, jobIdTwo);
DefaultAppScaleManager appScaleManager = new DefaultAppScaleManager(policyStore, mockAlarmClient, mockAppAutoScalingClient, v3JobOperations, new DefaultRegistry(), AutoScalingPolicyTests.mockAppScaleManagerConfiguration(), Schedulers.immediate(), mock(TitusRuntime.class));
AutoScalingPolicy autoScalingPolicyOne;
AutoScalingPolicy autoScalingPolicyTwo;
if (policyType == PolicyType.StepScaling) {
autoScalingPolicyOne = AutoScalingPolicyTests.buildStepScalingPolicy(jobIdOne);
autoScalingPolicyTwo = AutoScalingPolicyTests.buildStepScalingPolicy(jobIdTwo);
} else {
autoScalingPolicyOne = AutoScalingPolicyTests.buildTargetTrackingPolicy(jobIdOne);
autoScalingPolicyTwo = AutoScalingPolicyTests.buildTargetTrackingPolicy(jobIdTwo);
}
// call - createAutoScalingPolicy
appScaleManager.createAutoScalingPolicy(autoScalingPolicyOne).toBlocking().single();
String policyRefIdTwo = appScaleManager.createAutoScalingPolicy(autoScalingPolicyTwo).toBlocking().single();
AutoScalingPolicyTests.waitForCondition(() -> {
List<AutoScalingPolicy> policies = policyStore.retrievePolicies(false).toList().toBlocking().first();
return policies.size() == 2 && mockAppAutoScalingClient.getNumPolicies() == 2 && mockAppAutoScalingClient.getNumScalableTargets() == 2;
});
// verify counts in CloudAlarmClient, AppAutoScaleClient and AppScalePolicyStore
List<AutoScalingPolicy> policiesStored = policyStore.retrievePolicies(false).toList().toBlocking().first();
Assertions.assertThat(policiesStored.size()).isEqualTo(2);
Assertions.assertThat(mockAppAutoScalingClient.getNumPolicies()).isEqualTo(2);
Assertions.assertThat(mockAppAutoScalingClient.getNumScalableTargets()).isEqualTo(2);
if (policyType == PolicyType.StepScaling) {
Assertions.assertThat(mockAlarmClient.getNumOfAlarmsCreated()).isEqualTo(2);
}
appScaleManager.removeAutoScalingPolicy(policyRefIdTwo).await();
AutoScalingPolicyTests.waitForCondition(() -> {
List<AutoScalingPolicy> policies = policyStore.retrievePolicies(false).toList().toBlocking().first();
return policies.size() == 1 && mockAppAutoScalingClient.getNumPolicies() == 1 && mockAppAutoScalingClient.getNumScalableTargets() == 1;
});
// verify counts in CloudAlarmClient, AppAutoScaleClient and AppScalePolicyStore
policiesStored = policyStore.retrievePolicies(false).toList().toBlocking().first();
Assertions.assertThat(policiesStored.size()).isEqualTo(1);
Assertions.assertThat(mockAppAutoScalingClient.getNumPolicies()).isEqualTo(1);
Assertions.assertThat(mockAppAutoScalingClient.getNumScalableTargets()).isEqualTo(1);
if (policyType == PolicyType.StepScaling) {
Assertions.assertThat(mockAlarmClient.getNumOfAlarmsCreated()).isEqualTo(1);
}
}
Aggregations