Search in sources :

Example 1 with TaskQuery

use of com.netflix.titus.grpc.protogen.TaskQuery in project titus-control-plane by Netflix.

the class GatewayJobServiceGatewayTest method testCombineForFirstPage.

private TaskQueryResult testCombineForFirstPage() {
    TaskQuery taskQuery = TaskQuery.newBuilder().setPage(FIRST_PAGE).build();
    TaskQueryResult page0ActiveSetResult = takeActivePage(0);
    TaskQueryResult combinedResult = GatewayJobServiceGateway.combineTaskResults(taskQuery, page0ActiveSetResult, ARCHIVED_TASKS);
    checkCombinedResult(combinedResult, ARCHIVED_TASKS.subList(0, PAGE_SIZE));
    return combinedResult;
}
Also used : TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult)

Example 2 with TaskQuery

use of com.netflix.titus.grpc.protogen.TaskQuery in project titus-control-plane by Netflix.

the class DefaultAppAutoScalingCallbackService method getScalableTargetResourceInfo.

@Override
public Observable<ScalableTargetResourceInfo> getScalableTargetResourceInfo(String jobId, CallMetadata callMetadata) {
    TaskQuery taskQuery = TaskQuery.newBuilder().putFilteringCriteria("jobIds", jobId).putFilteringCriteria("taskStates", "Started").setPage(Page.newBuilder().setPageSize(1).build()).build();
    return jobServiceGateway.findTasks(taskQuery, callMetadata).map(taskQueryResult -> taskQueryResult.getPagination().getTotalItems()).flatMap(numStartedTasks -> jobServiceGateway.findJob(jobId, callMetadata).map(job -> Pair.of(job, numStartedTasks))).flatMap(jobTasksPair -> {
        Job job = jobTasksPair.getLeft();
        Integer numRunningTasks = jobTasksPair.getRight();
        if (!job.getJobDescriptor().hasService()) {
            return Observable.error(JobManagerException.notServiceJob(jobId));
        }
        ServiceJobSpec jobSpec = job.getJobDescriptor().getService();
        ScalableTargetResourceInfo.Builder scalableTargetResourceInfoBuilder = ScalableTargetResourceInfo.newBuilder().actualCapacity(jobTasksPair.getRight()).desiredCapacity(jobSpec.getCapacity().getDesired()).dimensionName(DIMENSION_NAME).resourceName(jobId).scalableTargetDimensionId(jobId).version(buildVersion(job));
        if (jobSpec.getCapacity().getDesired() != numRunningTasks) {
            scalableTargetResourceInfoBuilder.scalingStatus(ScalingStatus.InProgress.name());
        } else {
            scalableTargetResourceInfoBuilder.scalingStatus(ScalingStatus.Successful.name());
        }
        return Observable.just(scalableTargetResourceInfoBuilder.build());
    });
}
Also used : Logger(org.slf4j.Logger) ServiceJobSpec(com.netflix.titus.grpc.protogen.ServiceJobSpec) UInt32Value(com.google.protobuf.UInt32Value) JobStatus(com.netflix.titus.grpc.protogen.JobStatus) LoggerFactory(org.slf4j.LoggerFactory) Job(com.netflix.titus.grpc.protogen.Job) Singleton(javax.inject.Singleton) ArrayList(java.util.ArrayList) Page(com.netflix.titus.grpc.protogen.Page) Observable(rx.Observable) Inject(javax.inject.Inject) List(java.util.List) Pair(com.netflix.titus.common.util.tuple.Pair) Optional(java.util.Optional) JobManagerException(com.netflix.titus.api.jobmanager.service.JobManagerException) JobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.JobServiceGateway) JobCapacityWithOptionalAttributes(com.netflix.titus.grpc.protogen.JobCapacityWithOptionalAttributes) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) JobCapacityUpdateWithOptionalAttributes(com.netflix.titus.grpc.protogen.JobCapacityUpdateWithOptionalAttributes) ServiceJobSpec(com.netflix.titus.grpc.protogen.ServiceJobSpec) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Job(com.netflix.titus.grpc.protogen.Job)

Example 3 with TaskQuery

use of com.netflix.titus.grpc.protogen.TaskQuery in project titus-control-plane by Netflix.

the class AggregatingJobServiceGatewayWithSingleCellTest method findTasksMergesAllCellsIntoSingleResult.

@Test
public void findTasksMergesAllCellsIntoSingleResult() {
    List<Task> cellSnapshot = new ArrayList<>();
    // 10 jobs on each cell with TASKS_IN_GENERATED_JOBS tasks each
    for (int i = 0; i < 5; i++) {
        cellSnapshot.addAll(dataGenerator.newBatchJobWithTasks());
        cellSnapshot.addAll(dataGenerator.newServiceJobWithTasks());
        clock.advanceTime(1, TimeUnit.MINUTES);
    }
    cell.getServiceRegistry().addService(new CellWithFixedTasksService(cellSnapshot));
    TaskQuery query = TaskQuery.newBuilder().setPage(toGrpcPage(Page.unlimited())).build();
    final AssertableSubscriber<TaskQueryResult> testSubscriber = service.findTasks(query, UNDEFINED_CALL_METADATA).test();
    testSubscriber.awaitTerminalEvent(1, TimeUnit.SECONDS);
    testSubscriber.assertNoErrors().assertCompleted();
    testSubscriber.assertValueCount(1);
    final List<TaskQueryResult> results = testSubscriber.getOnNextEvents();
    assertThat(results).hasSize(1);
    // expect stackName to have changed
    List<Task> expected = cellSnapshot.stream().sorted(JobManagerCursors.taskCursorOrderComparator()).map(this::withStackName).collect(Collectors.toList());
    assertThat(results.get(0).getItemsList()).containsExactlyElementsOf(expected);
}
Also used : Task(com.netflix.titus.grpc.protogen.Task) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) ArrayList(java.util.ArrayList) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) Test(org.junit.Test)

Example 4 with TaskQuery

use of com.netflix.titus.grpc.protogen.TaskQuery in project titus-control-plane by Netflix.

the class GatewayJobServiceGateway method findTasks.

@Override
public Observable<TaskQueryResult> findTasks(TaskQuery taskQuery, CallMetadata callMetadata) {
    Map<String, String> filteringCriteriaMap = taskQuery.getFilteringCriteriaMap();
    Set<String> v3JobIds = new HashSet<>(StringExt.splitByComma(filteringCriteriaMap.getOrDefault("jobIds", "")));
    boolean needsMigrationFilter = "true".equalsIgnoreCase(filteringCriteriaMap.getOrDefault("needsMigration", "false"));
    // "needsMigration" query is served from the local job and relocation cache.
    if (needsMigrationFilter) {
        PageResult<Task> pageResult = needsMigrationQueryHandler.findTasks(GrpcJobQueryModelConverters.toJobQueryCriteria(taskQuery), toPage(taskQuery.getPage()));
        TaskQueryResult taskQueryResult = TaskQueryResult.newBuilder().setPagination(toGrpcPagination(pageResult.getPagination())).addAllItems(taskDataInjector.injectIntoTasks(pageResult.getItems())).build();
        return Observable.just(taskQueryResult);
    }
    Set<String> taskStates = Sets.newHashSet(StringExt.splitByComma(taskQuery.getFilteringCriteriaMap().getOrDefault("taskStates", "")));
    Observable<TaskQueryResult> observable;
    if (v3JobIds.isEmpty()) {
        // Active task set only
        observable = newActiveTaskQueryAction(taskQuery, callMetadata);
    } else {
        if (!taskStates.contains(TaskState.Finished.name())) {
            // Active task set only
            observable = newActiveTaskQueryAction(taskQuery, callMetadata);
        } else {
            Page page = taskQuery.getPage();
            boolean nextPageByNumber = StringExt.isEmpty(page.getCursor()) && page.getPageNumber() > 0;
            if (nextPageByNumber) {
                // In this case we ask for active and archived tasks using a page number > 0. Because of that
                // we have to fetch as much tasks from master as we can. Tasks that we do not fetch, will not be
                // visible to the client.
                TaskQuery largePageQuery = taskQuery.toBuilder().setPage(taskQuery.getPage().toBuilder().setPageNumber(0).setPageSize(gatewayConfiguration.getMaxTaskPageSize())).build();
                observable = newActiveTaskQueryAction(largePageQuery, callMetadata);
            } else {
                observable = newActiveTaskQueryAction(taskQuery, callMetadata);
            }
            observable = observable.flatMap(result -> retrieveArchivedTasksForJobs(v3JobIds, taskQuery).map(archivedTasks -> combineTaskResults(taskQuery, result, archivedTasks)));
        }
    }
    return observable.timeout(tunablesConfiguration.getRequestTimeoutMs(), TimeUnit.MILLISECONDS).map(queryResult -> taskDataInjector.injectIntoTaskQueryResult(queryResult));
}
Also used : JobStoreException(com.netflix.titus.api.jobmanager.store.JobStoreException) ObserveJobsQuery(com.netflix.titus.grpc.protogen.ObserveJobsQuery) TaskId(com.netflix.titus.grpc.protogen.TaskId) LoggerFactory(org.slf4j.LoggerFactory) StringExt(com.netflix.titus.common.util.StringExt) JobQueryResult(com.netflix.titus.grpc.protogen.JobQueryResult) GrpcUtil(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil) StreamObserver(io.grpc.stub.StreamObserver) TitusServiceException(com.netflix.titus.api.service.TitusServiceException) GrpcJobQueryModelConverters(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters) Map(java.util.Map) SanitizingJobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.SanitizingJobServiceGateway) Status(io.grpc.Status) JobStore(com.netflix.titus.api.jobmanager.store.JobStore) PaginationUtil(com.netflix.titus.api.model.PaginationUtil) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) JobDescriptor(com.netflix.titus.grpc.protogen.JobDescriptor) Predicate(java.util.function.Predicate) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) SECURITY_GROUPS_REQUIRED_FEATURE(com.netflix.titus.api.FeatureRolloutPlans.SECURITY_GROUPS_REQUIRED_FEATURE) Task(com.netflix.titus.grpc.protogen.Task) Set(java.util.Set) GrpcUtil.createRequestObservable(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil.createRequestObservable) JobFunctions(com.netflix.titus.api.jobmanager.model.job.JobFunctions) Collectors(java.util.stream.Collectors) TaskState(com.netflix.titus.api.jobmanager.model.job.TaskState) Sets(com.google.common.collect.Sets) Page(com.netflix.titus.grpc.protogen.Page) GrpcJobQueryModelConverters.toPage(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toPage) PageResult(com.netflix.titus.api.model.PageResult) List(java.util.List) GrpcJobQueryModelConverters.toGrpcPagination(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toGrpcPagination) GrpcUtil.createSimpleClientResponseObserver(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil.createSimpleClientResponseObserver) JobManagementServiceStub(com.netflix.titus.grpc.protogen.JobManagementServiceGrpc.JobManagementServiceStub) Clock(com.netflix.titus.common.util.time.Clock) JobChangeNotification(com.netflix.titus.grpc.protogen.JobChangeNotification) JobId(com.netflix.titus.grpc.protogen.JobId) EntitySanitizer(com.netflix.titus.common.model.sanitizer.EntitySanitizer) Job(com.netflix.titus.grpc.protogen.Job) JobManagerConfiguration(com.netflix.titus.runtime.jobmanager.JobManagerConfiguration) Singleton(javax.inject.Singleton) Function(java.util.function.Function) Observable(rx.Observable) HashSet(java.util.HashSet) Inject(javax.inject.Inject) JobQuery(com.netflix.titus.grpc.protogen.JobQuery) Pair(com.netflix.titus.common.util.tuple.Pair) TaskStatus(com.netflix.titus.grpc.protogen.TaskStatus) AdmissionSanitizer(com.netflix.titus.common.model.admission.AdmissionSanitizer) ExceptionExt(com.netflix.titus.common.util.ExceptionExt) JobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.JobServiceGateway) AdmissionValidator(com.netflix.titus.common.model.admission.AdmissionValidator) Named(javax.inject.Named) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) JobServiceGatewayDelegate(com.netflix.titus.runtime.jobmanager.gateway.JobServiceGatewayDelegate) Logger(org.slf4j.Logger) JobAssertions(com.netflix.titus.api.jobmanager.model.job.sanitizer.JobAssertions) LogStorageInfo(com.netflix.titus.api.jobmanager.model.job.LogStorageInfo) GrpcJobManagementModelConverters(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobManagementModelConverters) GrpcUtil.createWrappedStub(com.netflix.titus.runtime.endpoint.common.grpc.GrpcUtil.createWrappedStub) JOB_STRICT_SANITIZER(com.netflix.titus.api.jobmanager.model.job.sanitizer.JobSanitizerBuilder.JOB_STRICT_SANITIZER) TimeUnit(java.util.concurrent.TimeUnit) StatusRuntimeException(io.grpc.StatusRuntimeException) JobQueryCriteria(com.netflix.titus.runtime.endpoint.JobQueryCriteria) JobManagerCursors(com.netflix.titus.runtime.jobmanager.JobManagerCursors) GrpcJobQueryModelConverters.toJobQueryCriteria(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toJobQueryCriteria) GrpcRequestConfiguration(com.netflix.titus.runtime.connector.GrpcRequestConfiguration) GrpcJobServiceGateway(com.netflix.titus.runtime.jobmanager.gateway.GrpcJobServiceGateway) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Pagination(com.netflix.titus.api.model.Pagination) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) Collections(java.util.Collections) ENVIRONMENT_VARIABLE_NAMES_STRICT_VALIDATION_FEATURE(com.netflix.titus.api.FeatureRolloutPlans.ENVIRONMENT_VARIABLE_NAMES_STRICT_VALIDATION_FEATURE) Task(com.netflix.titus.grpc.protogen.Task) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Page(com.netflix.titus.grpc.protogen.Page) GrpcJobQueryModelConverters.toPage(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toPage) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) HashSet(java.util.HashSet)

Example 5 with TaskQuery

use of com.netflix.titus.grpc.protogen.TaskQuery in project titus-control-plane by Netflix.

the class CellWithFixedTasksService method findTasks.

@Override
public void findTasks(TaskQuery request, StreamObserver<TaskQueryResult> responseObserver) {
    Pair<List<Task>, Pagination> page = PaginationUtil.takePageWithCursor(toPage(request.getPage()), getTasksList(), JobManagerCursors.taskCursorOrderComparator(), JobManagerCursors::taskIndexOf, JobManagerCursors::newTaskCursorFrom);
    Set<String> fieldsFilter = new HashSet<>(request.getFieldsList());
    if (!fieldsFilter.isEmpty()) {
        fieldsFilter.add("id");
        page = page.mapLeft(tasks -> tasks.stream().map(task -> ProtobufExt.copy(task, fieldsFilter)).collect(Collectors.toList()));
    }
    TaskQueryResult result = TaskQueryResult.newBuilder().addAllItems(page.getLeft()).setPagination(toGrpcPagination(page.getRight())).build();
    responseObserver.onNext(result);
    responseObserver.onCompleted();
}
Also used : NOT_FOUND(io.grpc.Status.NOT_FOUND) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) Task(com.netflix.titus.grpc.protogen.Task) TaskId(com.netflix.titus.grpc.protogen.TaskId) Set(java.util.Set) JobManagementServiceGrpc(com.netflix.titus.grpc.protogen.JobManagementServiceGrpc) Function(java.util.function.Function) Collectors(java.util.stream.Collectors) ProtobufExt(com.netflix.titus.common.util.ProtobufExt) ArrayList(java.util.ArrayList) GrpcJobQueryModelConverters.toPage(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toPage) HashSet(java.util.HashSet) Empty(com.google.protobuf.Empty) List(java.util.List) GrpcJobQueryModelConverters.toGrpcPagination(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toGrpcPagination) StreamObserver(io.grpc.stub.StreamObserver) Pair(com.netflix.titus.common.util.tuple.Pair) JobManagerCursors(com.netflix.titus.runtime.jobmanager.JobManagerCursors) Map(java.util.Map) Pagination(com.netflix.titus.api.model.Pagination) Collections(java.util.Collections) PaginationUtil(com.netflix.titus.api.model.PaginationUtil) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) TaskKillRequest(com.netflix.titus.grpc.protogen.TaskKillRequest) GrpcJobQueryModelConverters.toGrpcPagination(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toGrpcPagination) Pagination(com.netflix.titus.api.model.Pagination) ArrayList(java.util.ArrayList) List(java.util.List) JobManagerCursors(com.netflix.titus.runtime.jobmanager.JobManagerCursors) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) HashSet(java.util.HashSet)

Aggregations

TaskQuery (com.netflix.titus.grpc.protogen.TaskQuery)9 TaskQueryResult (com.netflix.titus.grpc.protogen.TaskQueryResult)8 Task (com.netflix.titus.grpc.protogen.Task)5 Pair (com.netflix.titus.common.util.tuple.Pair)4 ArrayList (java.util.ArrayList)4 HashSet (java.util.HashSet)4 List (java.util.List)4 CallMetadata (com.netflix.titus.api.model.callmetadata.CallMetadata)3 Page (com.netflix.titus.grpc.protogen.Page)3 JobManagerCursors (com.netflix.titus.runtime.jobmanager.JobManagerCursors)3 Collections (java.util.Collections)3 Map (java.util.Map)3 Set (java.util.Set)3 Function (java.util.function.Function)3 Collectors (java.util.stream.Collectors)3 Inject (javax.inject.Inject)3 Singleton (javax.inject.Singleton)3 Test (org.junit.Test)3 Logger (org.slf4j.Logger)3 LoggerFactory (org.slf4j.LoggerFactory)3