Search in sources :

Example 11 with Page

use of com.netflix.titus.grpc.protogen.Page in project titus-control-plane by Netflix.

the class GatewayJobServiceGateway method findTasks.

@Override
public Observable<TaskQueryResult> findTasks(TaskQuery taskQuery, CallMetadata callMetadata) {
    Map<String, String> filteringCriteriaMap = taskQuery.getFilteringCriteriaMap();
    Set<String> v3JobIds = new HashSet<>(StringExt.splitByComma(filteringCriteriaMap.getOrDefault("jobIds", "")));
    boolean needsMigrationFilter = "true".equalsIgnoreCase(filteringCriteriaMap.getOrDefault("needsMigration", "false"));
    // "needsMigration" query is served from the local job and relocation cache.
    if (needsMigrationFilter) {
        PageResult<Task> pageResult = needsMigrationQueryHandler.findTasks(GrpcJobQueryModelConverters.toJobQueryCriteria(taskQuery), toPage(taskQuery.getPage()));
        TaskQueryResult taskQueryResult = TaskQueryResult.newBuilder().setPagination(toGrpcPagination(pageResult.getPagination())).addAllItems(taskDataInjector.injectIntoTasks(pageResult.getItems())).build();
        return Observable.just(taskQueryResult);
    }
    Set<String> taskStates = Sets.newHashSet(StringExt.splitByComma(taskQuery.getFilteringCriteriaMap().getOrDefault("taskStates", "")));
    Observable<TaskQueryResult> observable;
    if (v3JobIds.isEmpty()) {
        // Active task set only
        observable = newActiveTaskQueryAction(taskQuery, callMetadata);
    } else {
        if (!taskStates.contains(TaskState.Finished.name())) {
            // Active task set only
            observable = newActiveTaskQueryAction(taskQuery, callMetadata);
        } else {
            Page page = taskQuery.getPage();
            boolean nextPageByNumber = StringExt.isEmpty(page.getCursor()) && page.getPageNumber() > 0;
            if (nextPageByNumber) {
                // In this case we ask for active and archived tasks using a page number > 0. Because of that
                // we have to fetch as much tasks from master as we can. Tasks that we do not fetch, will not be
                // visible to the client.
                TaskQuery largePageQuery = taskQuery.toBuilder().setPage(taskQuery.getPage().toBuilder().setPageNumber(0).setPageSize(gatewayConfiguration.getMaxTaskPageSize())).build();
                observable = newActiveTaskQueryAction(largePageQuery, callMetadata);
            } else {
                observable = newActiveTaskQueryAction(taskQuery, callMetadata);
            }
            observable = observable.flatMap(result -> retrieveArchivedTasksForJobs(v3JobIds, taskQuery).map(archivedTasks -> combineTaskResults(taskQuery, result, archivedTasks)));
        }
    }
    return observable.timeout(tunablesConfiguration.getRequestTimeoutMs(), TimeUnit.MILLISECONDS).map(queryResult -> taskDataInjector.injectIntoTaskQueryResult(queryResult));
}
Also used : JobStoreException(com.netflix.titus.api.jobmanager.store.JobStoreException) ObserveJobsQuery(com.netflix.titus.grpc.protogen.ObserveJobsQuery) TaskId(com.netflix.titus.grpc.protogen.TaskId) LoggerFactory(org.slf4j.LoggerFactory) StringExt(com.netflix.titus.common.util.StringExt) JobQueryResult(com.netflix.titus.grpc.protogen.JobQueryResult) GrpcUtil(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil) StreamObserver(io.grpc.stub.StreamObserver) TitusServiceException(com.netflix.titus.api.service.TitusServiceException) GrpcJobQueryModelConverters(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters) Map(java.util.Map) SanitizingJobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.SanitizingJobServiceGateway) Status(io.grpc.Status) JobStore(com.netflix.titus.api.jobmanager.store.JobStore) PaginationUtil(com.netflix.titus.api.model.PaginationUtil) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) JobDescriptor(com.netflix.titus.grpc.protogen.JobDescriptor) Predicate(java.util.function.Predicate) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) SECURITY_GROUPS_REQUIRED_FEATURE(com.netflix.titus.api.FeatureRolloutPlans.SECURITY_GROUPS_REQUIRED_FEATURE) Task(com.netflix.titus.grpc.protogen.Task) Set(java.util.Set) GrpcUtil.createRequestObservable(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil.createRequestObservable) JobFunctions(com.netflix.titus.api.jobmanager.model.job.JobFunctions) Collectors(java.util.stream.Collectors) TaskState(com.netflix.titus.api.jobmanager.model.job.TaskState) Sets(com.google.common.collect.Sets) Page(com.netflix.titus.grpc.protogen.Page) GrpcJobQueryModelConverters.toPage(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toPage) PageResult(com.netflix.titus.api.model.PageResult) List(java.util.List) GrpcJobQueryModelConverters.toGrpcPagination(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toGrpcPagination) GrpcUtil.createSimpleClientResponseObserver(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil.createSimpleClientResponseObserver) JobManagementServiceStub(com.netflix.titus.grpc.protogen.JobManagementServiceGrpc.JobManagementServiceStub) Clock(com.netflix.titus.common.util.time.Clock) JobChangeNotification(com.netflix.titus.grpc.protogen.JobChangeNotification) JobId(com.netflix.titus.grpc.protogen.JobId) EntitySanitizer(com.netflix.titus.common.model.sanitizer.EntitySanitizer) Job(com.netflix.titus.grpc.protogen.Job) JobManagerConfiguration(com.netflix.titus.runtime.jobmanager.JobManagerConfiguration) Singleton(javax.inject.Singleton) Function(java.util.function.Function) Observable(rx.Observable) HashSet(java.util.HashSet) Inject(javax.inject.Inject) JobQuery(com.netflix.titus.grpc.protogen.JobQuery) Pair(com.netflix.titus.common.util.tuple.Pair) TaskStatus(com.netflix.titus.grpc.protogen.TaskStatus) AdmissionSanitizer(com.netflix.titus.common.model.admission.AdmissionSanitizer) ExceptionExt(com.netflix.titus.common.util.ExceptionExt) JobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.JobServiceGateway) AdmissionValidator(com.netflix.titus.common.model.admission.AdmissionValidator) Named(javax.inject.Named) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) JobServiceGatewayDelegate(com.netflix.titus.runtime.jobmanager.gateway.JobServiceGatewayDelegate) Logger(org.slf4j.Logger) JobAssertions(com.netflix.titus.api.jobmanager.model.job.sanitizer.JobAssertions) LogStorageInfo(com.netflix.titus.api.jobmanager.model.job.LogStorageInfo) GrpcJobManagementModelConverters(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobManagementModelConverters) GrpcUtil.createWrappedStub(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil.createWrappedStub) JOB_STRICT_SANITIZER(com.netflix.titus.api.jobmanager.model.job.sanitizer.JobSanitizerBuilder.JOB_STRICT_SANITIZER) TimeUnit(java.util.concurrent.TimeUnit) StatusRuntimeException(io.grpc.StatusRuntimeException) JobQueryCriteria(com.netflix.titus.runtime.endpoint.JobQueryCriteria) JobManagerCursors(com.netflix.titus.runtime.jobmanager.JobManagerCursors) GrpcJobQueryModelConverters.toJobQueryCriteria(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toJobQueryCriteria) GrpcRequestConfiguration(com.netflix.titus.runtime.connector.GrpcRequestConfiguration) GrpcJobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.GrpcJobServiceGateway) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Pagination(com.netflix.titus.api.model.Pagination) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) Collections(java.util.Collections) ENVIRONMENT_VARIABLE_NAMES_STRICT_VALIDATION_FEATURE(com.netflix.titus.api.FeatureRolloutPlans.ENVIRONMENT_VARIABLE_NAMES_STRICT_VALIDATION_FEATURE) Task(com.netflix.titus.grpc.protogen.Task) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Page(com.netflix.titus.grpc.protogen.Page) GrpcJobQueryModelConverters.toPage(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toPage) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) HashSet(java.util.HashSet)

Example 12 with Page

use of com.netflix.titus.grpc.protogen.Page in project titus-control-plane by Netflix.

the class LocalCacheQueryProcessorTest method testFindTasks.

@Test
public void testFindTasks() {
    List<Task> tasks1 = addToJobDataReplicator(newJobAndTasks("job1", 2)).getRight();
    List<Task> tasks2 = addToJobDataReplicator(newJobAndTasks("job2", 4)).getRight();
    List<Task> tasks3 = addToJobDataReplicator(newJobAndTasks("job3", 0)).getRight();
    // Expect three pages
    TaskQueryResult page1Result = processor.findTasks(TASK_QUERY_ALL_WITH_PAGE_SIZE_2);
    assertThat(page1Result.getItemsList()).hasSize(2);
    Page page2 = Page.newBuilder().setPageSize(2).setCursor(page1Result.getPagination().getCursor()).build();
    TaskQueryResult page2Result = processor.findTasks(TaskQuery.newBuilder().setPage(page2).build());
    assertThat(page2Result.getItemsList()).hasSize(2);
    Page page3 = Page.newBuilder().setPageSize(2).setCursor(page2Result.getPagination().getCursor()).build();
    TaskQueryResult page3Result = processor.findTasks(TaskQuery.newBuilder().setPage(page3).build());
    assertThat(page3Result.getItemsList()).hasSize(2);
    assertThat(page3Result.getPagination().getHasMore()).isFalse();
    List<String> taskIds = CollectionsExt.merge(page1Result.getItemsList().stream().map(com.netflix.titus.grpc.protogen.Task::getId).collect(Collectors.toList()), page2Result.getItemsList().stream().map(com.netflix.titus.grpc.protogen.Task::getId).collect(Collectors.toList()), page3Result.getItemsList().stream().map(com.netflix.titus.grpc.protogen.Task::getId).collect(Collectors.toList()));
    List<String> expectedTaskIds = CollectionsExt.merge(tasks1.stream().map(Task::getId).collect(Collectors.toList()), tasks2.stream().map(Task::getId).collect(Collectors.toList()), tasks3.stream().map(Task::getId).collect(Collectors.toList()));
    assertThat(taskIds).containsAll(expectedTaskIds);
}
Also used : Task(com.netflix.titus.api.jobmanager.model.job.Task) Page(com.netflix.titus.grpc.protogen.Page) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) Test(org.junit.Test)

Example 13 with Page

use of com.netflix.titus.grpc.protogen.Page in project titus-control-plane by Netflix.

the class JobCriteriaQueryTest method testPagination.

@Test(timeout = 30_000)
public void testPagination() {
    // We have 3 batch and 3 service jobs.
    Page firstPageOf5 = Page.newBuilder().setPageNumber(0).setPageSize(5).build();
    Page secondPageOf5 = Page.newBuilder().setPageNumber(1).setPageSize(5).build();
    // Jobs
    JobQuery.Builder jobQueryBuilder = JobQuery.newBuilder().putFilteringCriteria("attributes", PRE_CREATED_JOBS_LABEL);
    JobQueryResult jobQueryResult = client.findJobs(jobQueryBuilder.setPage(firstPageOf5).build());
    assertThat(jobQueryResult.getItemsList()).hasSize(5);
    checkPage(jobQueryResult.getPagination(), firstPageOf5, 2, 6, true);
    JobQueryResult jobQueryResult2 = client.findJobs(jobQueryBuilder.setPage(secondPageOf5).build());
    assertThat(jobQueryResult2.getItemsList()).hasSize(1);
    checkPage(jobQueryResult2.getPagination(), secondPageOf5, 2, 6, false);
    Set<String> foundJobIds = new HashSet<>();
    jobQueryResult.getItemsList().forEach(j -> foundJobIds.add(j.getId()));
    jobQueryResult2.getItemsList().forEach(j -> foundJobIds.add(j.getId()));
    assertThat(foundJobIds).hasSize(6);
    // Tasks
    TaskQuery.Builder taskQueryBuilder = TaskQuery.newBuilder().putFilteringCriteria("attributes", PRE_CREATED_JOBS_LABEL);
    TaskQueryResult taskQueryResult = client.findTasks(taskQueryBuilder.setPage(firstPageOf5).build());
    assertThat(taskQueryResult.getItemsList()).hasSize(5);
    checkPage(taskQueryResult.getPagination(), firstPageOf5, 2, 6, true);
    TaskQueryResult taskQueryResult2 = client.findTasks(taskQueryBuilder.setPage(secondPageOf5).build());
    assertThat(taskQueryResult2.getItemsList()).hasSize(1);
    checkPage(taskQueryResult2.getPagination(), secondPageOf5, 2, 6, false);
    Set<String> foundTasksIds = new HashSet<>();
    taskQueryResult.getItemsList().forEach(j -> foundTasksIds.add(j.getId()));
    taskQueryResult2.getItemsList().forEach(j -> foundTasksIds.add(j.getId()));
    assertThat(foundTasksIds).hasSize(6);
}
Also used : TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Page(com.netflix.titus.grpc.protogen.Page) JobQuery(com.netflix.titus.grpc.protogen.JobQuery) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) JobQueryResult(com.netflix.titus.grpc.protogen.JobQueryResult) HashSet(java.util.HashSet) BaseIntegrationTest(com.netflix.titus.master.integration.BaseIntegrationTest) IntegrationTest(com.netflix.titus.testkit.junit.category.IntegrationTest) Test(org.junit.Test)

Example 14 with Page

use of com.netflix.titus.grpc.protogen.Page in project titus-control-plane by Netflix.

the class JobCursorArchiveQueryTest method testArchiveQuery.

@Test(timeout = TEST_TIMEOUT_MS)
public void testArchiveQuery() {
    Evaluators.times(TASKS_PER_JOB, idx -> jobScenarioBuilder.getTaskByIndex(idx).killTask().completeKillInitiated().expectStateUpdateSkipOther(TaskStatus.TaskState.Finished));
    Evaluators.times(TASKS_PER_JOB, idx -> jobScenarioBuilder.expectTaskInSlot(idx, 1));
    int allTasksCount = TASKS_PER_JOB * 2;
    Set<String> taskIds = new HashSet<>();
    Page current = TWO_ITEM_PAGE;
    for (int i = 0; i < TASKS_PER_JOB; i++) {
        TaskQuery query = TaskQuery.newBuilder().setPage(current).putFilteringCriteria("jobIds", jobScenarioBuilder.getJobId()).putFilteringCriteria("taskStates", ALL_TASK_STATES).build();
        TaskQueryResult result = client.findTasks(query);
        assertThat(result.getPagination().getTotalItems()).isEqualTo(allTasksCount);
        assertThat(result.getPagination().getCursor()).isNotEmpty();
        taskIds.addAll(result.getItemsList().stream().map(Task::getId).collect(Collectors.toList()));
        current = query.getPage().toBuilder().setCursor(result.getPagination().getCursor()).build();
    }
    assertThat(taskIds).hasSize(allTasksCount);
}
Also used : Task(com.netflix.titus.grpc.protogen.Task) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Page(com.netflix.titus.grpc.protogen.Page) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) HashSet(java.util.HashSet) BaseIntegrationTest(com.netflix.titus.master.integration.BaseIntegrationTest) IntegrationTest(com.netflix.titus.testkit.junit.category.IntegrationTest) Test(org.junit.Test)

Aggregations

Page (com.netflix.titus.grpc.protogen.Page)14 TaskQuery (com.netflix.titus.grpc.protogen.TaskQuery)7 JobQuery (com.netflix.titus.grpc.protogen.JobQuery)6 TaskQueryResult (com.netflix.titus.grpc.protogen.TaskQueryResult)6 Test (org.junit.Test)6 JobQueryResult (com.netflix.titus.grpc.protogen.JobQueryResult)5 ApiOperation (io.swagger.annotations.ApiOperation)5 CallMetadata (com.netflix.titus.api.model.callmetadata.CallMetadata)4 Task (com.netflix.titus.grpc.protogen.Task)4 BaseIntegrationTest (com.netflix.titus.master.integration.BaseIntegrationTest)4 IntegrationTest (com.netflix.titus.testkit.junit.category.IntegrationTest)4 JobFunctions (com.netflix.titus.api.jobmanager.model.job.JobFunctions)3 Job (com.netflix.titus.grpc.protogen.Job)3 HashSet (java.util.HashSet)3 List (java.util.List)3 GetMapping (org.springframework.web.bind.annotation.GetMapping)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 Image (com.netflix.titus.api.jobmanager.model.job.Image)2 JobDescriptor (com.netflix.titus.api.jobmanager.model.job.JobDescriptor)2 JobGroupInfo (com.netflix.titus.api.jobmanager.model.job.JobGroupInfo)2