Examples with JobScenarioBuilder com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder used on opensource projects

Example 1 with JobScenarioBuilder

use of com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder in project titus-control-plane by Netflix.

the class V3JobSchedulingAndRebootTest method submitBatchJobAndRebootTitusMaster.

@Test(timeout = LONG_TEST_TIMEOUT_MS)
public void submitBatchJobAndRebootTitusMaster() {
    jobsScenarioBuilder.schedule(ONE_TASK_BATCH_JOB, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.startTasksInNewJob()));
    jobsScenarioBuilder.stop();
    titusStackResource.getMaster().reboot();
    JobsScenarioBuilder newJobsScenarioBuilder = new JobsScenarioBuilder(titusStackResource.getOperations());
    newJobsScenarioBuilder.assertJobs(jobs -> jobs.size() == 1).takeJob(0).assertJob(JobAsserts.jobInState(JobState.Accepted)).assertTasks(tasks -> tasks.size() == 1);
}

Example 2 with JobScenarioBuilder

use of com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder in project titus-control-plane by Netflix.

the class JobIpAllocationsTest method testWaitingTaskContext.

/**
 * Tests a job waiting for an in use IP allocation has updated task context fields.
 */
@Test(timeout = 30_000)
// TODO Read static IP allocation status from pod message field, and add it to the task context.
@Ignore
public void testWaitingTaskContext() throws Exception {
    JobDescriptor<ServiceJobExt> firstIpJobDescriptor = ONE_TASK_SERVICE_JOB;
    JobDescriptor<ServiceJobExt> secondIpJobDescriptor = firstIpJobDescriptor.but(j -> j.getJobGroupInfo().toBuilder().withSequence("v001"));
    // Schedule the first task and ensure it's in the correct zone with the correct task context
    jobsScenarioBuilder.schedule(firstIpJobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.startTasksInNewJob()).allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectTaskContext(TaskAttributes.TASK_ATTRIBUTES_IP_ALLOCATION_ID, getIpAllocationIdFromJob(0, firstIpJobDescriptor))).allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectZoneId(getZoneFromJobIpAllocation(0, firstIpJobDescriptor))));
    String firstJobId = jobsScenarioBuilder.takeJob(0).getJobId();
    // Schedule the second task and ensure it's blocked on the first task
    jobsScenarioBuilder.schedule(secondIpJobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.jobAccepted()).expectAllTasksCreated().allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectStateUpdates(TaskStatus.TaskState.Accepted)));
    String secondJobId = jobsScenarioBuilder.takeJob(1).getJobId();
    // Query the gRPC endpoint and ensure the first task does not have a waiting task context field.
    TaskQueryResult firstTaskQueryResult = client.findTasks(TaskQuery.newBuilder().setPage(Page.newBuilder().setPageSize(100).build()).putFilteringCriteria("jobIds", firstJobId).build());
    assertThat(firstTaskQueryResult.getItemsCount()).isEqualTo(1);
    firstTaskQueryResult.getItemsList().forEach(task -> {
        assertThat(task.getTaskContextMap()).doesNotContainKeys(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION);
    });
    String firstTaskId = firstTaskQueryResult.getItems(0).getId();
    // Query the gRPC endpoint and ensure the second task has a waiting task context field.
    TaskQueryResult secondTaskQueryResult = client.findTasks(TaskQuery.newBuilder().setPage(Page.newBuilder().setPageSize(100).build()).putFilteringCriteria("jobIds", secondJobId).build());
    assertThat(secondTaskQueryResult.getItemsCount()).isEqualTo(1);
    secondTaskQueryResult.getItemsList().forEach(task -> {
        assertThat(task.getTaskContextMap()).contains(new AbstractMap.SimpleImmutableEntry<>(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION, firstTaskId));
    });
    // Observe the second job and ensure the streamed task has a waiting task context field.
    boolean verified = false;
    Iterator<JobChangeNotification> it = client.observeJob(JobId.newBuilder().setId(secondJobId).build());
    while (it.hasNext()) {
        JobChangeNotification jobChangeNotification = it.next();
        if (jobChangeNotification.hasTaskUpdate()) {
            Map<String, String> taskContext = jobChangeNotification.getTaskUpdate().getTask().getTaskContextMap();
            assertThat(taskContext).contains(new AbstractMap.SimpleImmutableEntry<>(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION, firstTaskId));
            verified = true;
        } else if (jobChangeNotification.hasSnapshotEnd()) {
            break;
        }
    }
    assertThat(verified).isTrue();
}

Example 3 with JobScenarioBuilder

use of com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder in project titus-control-plane by Netflix.

the class JobCriteriaQueryTest method testSearchByJobState.

@Test(timeout = 30_000)
public void testSearchByJobState() throws Exception {
    JobDescriptor<BatchJobExt> jobDescriptor = batchJobDescriptors().getValue().toBuilder().withApplicationName("testSearchByJobState").build();
    String acceptedJobId = jobsScenarioBuilder.scheduleAndReturnJob(jobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob())).getId();
    String killInitiatedJobId = jobsScenarioBuilder.scheduleAndReturnJob(jobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob()).killJob().expectJobUpdateEvent(job -> job.getStatus().getState() == JobState.KillInitiated, "Expected state: " + JobState.KillInitiated)).getId();
    String acceptedTaskId = jobsScenarioBuilder.takeJob(acceptedJobId).getTaskByIndex(0).getTask().getId();
    String killInitiatedTaskId = jobsScenarioBuilder.takeJob(killInitiatedJobId).getTaskByIndex(0).getTask().getId();
    // Indexes are recomputed after events are sent, so if we run findJobs/findTasks immediately, they may use stale index.
    Thread.sleep(10);
    JobQuery.Builder jobQueryBuilder = JobQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByJobState").setPage(PAGE);
    TaskQuery.Builder taskQueryBuilder = TaskQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByJobState").setPage(PAGE);
    // Jobs (Accepted)
    JobQueryResult acceptedJobQueryResult = client.findJobs(jobQueryBuilder.putFilteringCriteria("jobState", "Accepted").build());
    assertThat(acceptedJobQueryResult.getItemsList()).hasSize(1);
    Job acceptedJobQueryResultItem = acceptedJobQueryResult.getItems(0);
    assertThat(acceptedJobQueryResultItem.getId()).isEqualTo(acceptedJobId);
    // Jobs (KillInitiated)
    JobQueryResult killInitJobQueryResult = client.findJobs(jobQueryBuilder.putFilteringCriteria("jobState", "KillInitiated").setPage(PAGE).build());
    assertThat(killInitJobQueryResult.getItemsList()).hasSize(1);
    Job killInitJobQueryResultItem = killInitJobQueryResult.getItems(0);
    assertThat(killInitJobQueryResultItem.getId()).isEqualTo(killInitiatedJobId);
    // Tasks (Accepted)
    TaskQueryResult acceptedTaskQueryResult = client.findTasks(taskQueryBuilder.putFilteringCriteria("jobState", "Accepted").setPage(PAGE).build());
    assertThat(acceptedTaskQueryResult.getItemsList()).hasSize(1);
    assertThat(acceptedTaskQueryResult.getItems(0).getId()).isEqualTo(acceptedTaskId);
    // Tasks (KillInitiated)
    TaskQueryResult killInitTaskQueryResult = client.findTasks(taskQueryBuilder.putFilteringCriteria("jobState", "KillInitiated").setPage(PAGE).build());
    assertThat(killInitTaskQueryResult.getItemsList()).hasSize(1);
    assertThat(killInitTaskQueryResult.getItems(0).getId()).isEqualTo(killInitiatedTaskId);
}

Example 4 with JobScenarioBuilder

use of com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder in project titus-control-plane by Netflix.

the class JobCriteriaQueryTest method testSearchByCellV3.

@Test(timeout = 30_000)
public void testSearchByCellV3() {
    final int numberOfJobs = 3;
    String[] expectedJobIds = new String[numberOfJobs];
    String[] expectedTaskIds = new String[numberOfJobs];
    for (int i = 0; i < numberOfJobs; i++) {
        String jobId = jobsScenarioBuilder.scheduleAndReturnJob(BATCH_JOB_TEMPLATE.toBuilder().withApplicationName("testSearchByCellV3").build(), jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob())).getId();
        expectedJobIds[i] = jobId;
        expectedTaskIds[i] = jobsScenarioBuilder.takeJob(jobId).getTaskByIndex(0).getTask().getId();
    }
    // Jobs
    JobQuery.Builder jobQueryBuilder = JobQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByCellV3").setPage(PAGE);
    JobQueryResult jobs1 = client.findJobs(jobQueryBuilder.putFilteringCriteria("attributes", "titus.cell,titus.stack").putFilteringCriteria("attributes.op", "or").build());
    assertContainsJobs(jobs1, expectedJobIds);
    jobs1.getItemsList().forEach(job -> assertCellInfo(job, EmbeddedTitusMaster.CELL_NAME));
    JobQueryResult jobs2 = client.findJobs(jobQueryBuilder.putFilteringCriteria("attributes", "titus.cell").putFilteringCriteria("attributes.op", "and").build());
    assertContainsJobs(jobs2, expectedJobIds);
    jobs2.getItemsList().forEach(job -> assertCellInfo(job, EmbeddedTitusMaster.CELL_NAME));
    JobQueryResult jobs3 = client.findJobs(jobQueryBuilder.putFilteringCriteria("attributes", String.format("titus.cell:%1$s,titus.stack:%1$s", EmbeddedTitusMaster.CELL_NAME)).putFilteringCriteria("attributes.op", "or").build());
    assertContainsJobs(jobs3, expectedJobIds);
    jobs3.getItemsList().forEach(job -> assertCellInfo(job, EmbeddedTitusMaster.CELL_NAME));
    JobQueryResult jobs4 = client.findJobs(jobQueryBuilder.putFilteringCriteria("attributes", String.format("titus.cell:%1$s", EmbeddedTitusMaster.CELL_NAME)).putFilteringCriteria("attributes.op", "and").build());
    assertContainsJobs(jobs4, expectedJobIds);
    jobs4.getItemsList().forEach(job -> assertCellInfo(job, EmbeddedTitusMaster.CELL_NAME));
    // Tasks
    TaskQuery.Builder taskQueryBuilder = TaskQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByCellV3").setPage(PAGE);
    TaskQueryResult tasks1 = client.findTasks(taskQueryBuilder.putFilteringCriteria("attributes", "titus.cell,titus.stack").putFilteringCriteria("attributes.op", "or").build());
    assertContainsTasks(tasks1, expectedTaskIds);
    tasks1.getItemsList().forEach(task -> assertCellInfo(task, EmbeddedTitusMaster.CELL_NAME));
    TaskQueryResult tasks2 = client.findTasks(taskQueryBuilder.putFilteringCriteria("attributes", "titus.cell").putFilteringCriteria("attributes.op", "and").build());
    assertContainsTasks(tasks2, expectedTaskIds);
    tasks2.getItemsList().forEach(task -> assertCellInfo(task, EmbeddedTitusMaster.CELL_NAME));
    TaskQueryResult tasks3 = client.findTasks(taskQueryBuilder.putFilteringCriteria("attributes", String.format("titus.cell:%1$s,titus.stack:%1$s", EmbeddedTitusMaster.CELL_NAME)).putFilteringCriteria("attributes.op", "or").build());
    assertContainsTasks(tasks3, expectedTaskIds);
    tasks3.getItemsList().forEach(task -> assertCellInfo(task, EmbeddedTitusMaster.CELL_NAME));
    final TaskQueryResult tasks4 = client.findTasks(taskQueryBuilder.putFilteringCriteria("attributes", String.format("titus.cell:%1$s", EmbeddedTitusMaster.CELL_NAME)).putFilteringCriteria("attributes.op", "and").build());
    assertContainsTasks(tasks4, expectedTaskIds);
    tasks4.getItemsList().forEach(task -> assertCellInfo(task, EmbeddedTitusMaster.CELL_NAME));
}

Example 5 with JobScenarioBuilder

use of com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder in project titus-control-plane by Netflix.

the class JobObserveTest method observeJobs.

@Test(timeout = LONG_TEST_TIMEOUT_MS)
public void observeJobs() throws InterruptedException {
    TestStreamObserver<JobChangeNotification> subscriber = subscribe(ObserveJobsQuery.getDefaultInstance());
    for (int i = 0; i < 2; i++) {
        String jobId = jobsScenarioBuilder.scheduleAndReturnJob(oneTaskBatchJobDescriptor(), jobScenarioBuilder -> jobScenarioBuilder.template(startTasksInNewJob()).template(ScenarioTemplates.killJob())).getId();
        JobChangeNotification event;
        while (true) {
            event = subscriber.takeNext(SHORT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
            if (event.hasJobUpdate()) {
                Job job = event.getJobUpdate().getJob();
                assertThat(job.getId()).isEqualTo(jobId);
                CellAssertions.assertCellInfo(job, EmbeddedTitusMaster.CELL_NAME);
                if (job.getStatus().getState() == JobState.Finished) {
                    break;
                }
            }
        }
    }
    subscriber.cancel();
}