Search in sources :

Example 1 with TaskQueryResult

use of com.netflix.titus.grpc.protogen.TaskQueryResult in project titus-control-plane by Netflix.

the class AggregatingJobServiceGatewayTest method findTasksWithFieldFiltering.

@Test
public void findTasksWithFieldFiltering() {
    Pair<List<Task>, List<Task>> cellSnapshots = generateTestJobsWithTasks();
    List<Task> allTasks = walkAllPages(6, request -> service.findTasks(request, UNDEFINED_CALL_METADATA), page -> TaskQuery.newBuilder().setPage(page).addFields("jobId").build(), TaskQueryResult::getPagination, TaskQueryResult::getItemsList);
    assertThat(allTasks).hasSize(cellSnapshots.getLeft().size() + cellSnapshots.getRight().size());
    for (Task task : allTasks) {
        assertThat(task.getId()).isNotEmpty();
        assertThat(task.getJobId()).isNotEmpty();
        assertThat(task.getStatus().getReasonMessage()).isEmpty();
        assertThat(task.getStatusHistoryList()).isEmpty();
    }
}
Also used : Task(com.netflix.titus.grpc.protogen.Task) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) Test(org.junit.Test)

Example 2 with TaskQueryResult

use of com.netflix.titus.grpc.protogen.TaskQueryResult in project titus-control-plane by Netflix.

the class TasksGetCommand method execute.

@Override
public void execute(CommandContext context) throws Exception {
    CommandLine cli = context.getCLI();
    TaskQuery.Builder queryBuilder = TaskQuery.newBuilder();
    Page.Builder pageBuilder = Page.newBuilder();
    if (cli.hasOption('s')) {
        pageBuilder.setPageSize(((Number) cli.getParsedOptionValue("s")).intValue());
    } else {
        pageBuilder.setPageSize(100);
    }
    if (cli.hasOption('n')) {
        pageBuilder.setPageNumber(((Number) cli.getParsedOptionValue("n")).intValue());
    }
    queryBuilder.setPage(pageBuilder);
    if (cli.hasOption('f')) {
        queryBuilder.addAllFields(StringExt.splitByComma(cli.getOptionValue('f')));
    }
    try {
        TaskQueryResult result = context.getJobManagementGrpcBlockingStub().findTasks(queryBuilder.build());
        logger.info("Found tasks: " + PrettyPrinters.print(result));
    } catch (Exception e) {
        ErrorReports.handleReplyError("Command execution error", e);
    }
}
Also used : CommandLine(org.apache.commons.cli.CommandLine) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Page(com.netflix.titus.grpc.protogen.Page) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult)

Example 3 with TaskQueryResult

use of com.netflix.titus.grpc.protogen.TaskQueryResult in project titus-control-plane by Netflix.

the class DefaultJobManagementServiceGrpc method findTasks.

@Override
public void findTasks(TaskQuery taskQuery, StreamObserver<TaskQueryResult> responseObserver) {
    if (!checkPageIsValid(taskQuery.getPage(), responseObserver)) {
        return;
    }
    try {
        // We need to find all tasks to get the total number of them.
        List<com.netflix.titus.api.jobmanager.model.job.Task> allFilteredTasks = new ArrayList<>();
        for (Pair<com.netflix.titus.api.jobmanager.model.job.Job<?>, com.netflix.titus.api.jobmanager.model.job.Task> jobTaskPair : jobOperations.findTasks(new V3TaskQueryCriteriaEvaluator(toJobQueryCriteria(taskQuery), titusRuntime), 0, Integer.MAX_VALUE / 2)) {
            com.netflix.titus.api.jobmanager.model.job.Task right = jobTaskPair.getRight();
            allFilteredTasks.add(right);
        }
        Pair<List<com.netflix.titus.api.jobmanager.model.job.Task>, Pagination> queryResult = PaginationUtil.takePageWithCursorAndKeyExtractor(toPage(taskQuery.getPage()), allFilteredTasks, JobComparators::createTaskKeyOf, JobManagerCursors::coreTaskIndexOf, JobManagerCursors::newTaskCoreCursorFrom);
        List<Task> grpcTasks = new ArrayList<>();
        for (com.netflix.titus.api.jobmanager.model.job.Task task : queryResult.getLeft()) {
            Task toGrpcTask = grpcObjectsCache.getTask(task);
            grpcTasks.add(toGrpcTask);
        }
        TaskQueryResult grpcQueryResult;
        if (taskQuery.getFieldsList().isEmpty()) {
            grpcQueryResult = toTaskQueryResult(grpcTasks, queryResult.getRight());
        } else {
            Set<String> fields = new HashSet<>(taskQuery.getFieldsList());
            fields.addAll(TASK_MINIMUM_FIELD_SET);
            List<Task> filtered = new ArrayList<>();
            for (Task t : grpcTasks) {
                filtered.add(ProtobufExt.copy(t, fields));
            }
            grpcQueryResult = toTaskQueryResult(filtered, queryResult.getRight());
        }
        responseObserver.onNext(grpcQueryResult);
        responseObserver.onCompleted();
    } catch (Exception e) {
        safeOnError(logger, e, responseObserver);
    }
}
Also used : Task(com.netflix.titus.grpc.protogen.Task) V3TaskQueryCriteriaEvaluator(com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator) ArrayList(java.util.ArrayList) JobManagerCursors(com.netflix.titus.runtime.jobmanager.JobManagerCursors) Pagination(com.netflix.titus.api.model.Pagination) GrpcJobQueryModelConverters.toGrpcPagination(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toGrpcPagination) JobComparators(com.netflix.titus.runtime.jobmanager.JobComparators) ArrayList(java.util.ArrayList) List(java.util.List) Job(com.netflix.titus.grpc.protogen.Job) HashSet(java.util.HashSet) StatusRuntimeException(io.grpc.StatusRuntimeException) TitusServiceException(com.netflix.titus.api.service.TitusServiceException) JobManagerException(com.netflix.titus.api.jobmanager.service.JobManagerException) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult)

Example 4 with TaskQueryResult

use of com.netflix.titus.grpc.protogen.TaskQueryResult in project titus-control-plane by Netflix.

the class JobIpAllocationsTest method testWaitingTaskContext.

/**
 * Tests a job waiting for an in use IP allocation has updated task context fields.
 */
@Test(timeout = 30_000)
// TODO Read static IP allocation status from pod message field, and add it to the task context.
@Ignore
public void testWaitingTaskContext() throws Exception {
    JobDescriptor<ServiceJobExt> firstIpJobDescriptor = ONE_TASK_SERVICE_JOB;
    JobDescriptor<ServiceJobExt> secondIpJobDescriptor = firstIpJobDescriptor.but(j -> j.getJobGroupInfo().toBuilder().withSequence("v001"));
    // Schedule the first task and ensure it's in the correct zone with the correct task context
    jobsScenarioBuilder.schedule(firstIpJobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.startTasksInNewJob()).allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectTaskContext(TaskAttributes.TASK_ATTRIBUTES_IP_ALLOCATION_ID, getIpAllocationIdFromJob(0, firstIpJobDescriptor))).allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectZoneId(getZoneFromJobIpAllocation(0, firstIpJobDescriptor))));
    String firstJobId = jobsScenarioBuilder.takeJob(0).getJobId();
    // Schedule the second task and ensure it's blocked on the first task
    jobsScenarioBuilder.schedule(secondIpJobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.jobAccepted()).expectAllTasksCreated().allTasks(taskScenarioBuilder -> taskScenarioBuilder.expectStateUpdates(TaskStatus.TaskState.Accepted)));
    String secondJobId = jobsScenarioBuilder.takeJob(1).getJobId();
    // Query the gRPC endpoint and ensure the first task does not have a waiting task context field.
    TaskQueryResult firstTaskQueryResult = client.findTasks(TaskQuery.newBuilder().setPage(Page.newBuilder().setPageSize(100).build()).putFilteringCriteria("jobIds", firstJobId).build());
    assertThat(firstTaskQueryResult.getItemsCount()).isEqualTo(1);
    firstTaskQueryResult.getItemsList().forEach(task -> {
        assertThat(task.getTaskContextMap()).doesNotContainKeys(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION);
    });
    String firstTaskId = firstTaskQueryResult.getItems(0).getId();
    // Query the gRPC endpoint and ensure the second task has a waiting task context field.
    TaskQueryResult secondTaskQueryResult = client.findTasks(TaskQuery.newBuilder().setPage(Page.newBuilder().setPageSize(100).build()).putFilteringCriteria("jobIds", secondJobId).build());
    assertThat(secondTaskQueryResult.getItemsCount()).isEqualTo(1);
    secondTaskQueryResult.getItemsList().forEach(task -> {
        assertThat(task.getTaskContextMap()).contains(new AbstractMap.SimpleImmutableEntry<>(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION, firstTaskId));
    });
    // Observe the second job and ensure the streamed task has a waiting task context field.
    boolean verified = false;
    Iterator<JobChangeNotification> it = client.observeJob(JobId.newBuilder().setId(secondJobId).build());
    while (it.hasNext()) {
        JobChangeNotification jobChangeNotification = it.next();
        if (jobChangeNotification.hasTaskUpdate()) {
            Map<String, String> taskContext = jobChangeNotification.getTaskUpdate().getTask().getTaskContextMap();
            assertThat(taskContext).contains(new AbstractMap.SimpleImmutableEntry<>(TaskAttributes.TASK_ATTRIBUTES_IN_USE_IP_ALLOCATION, firstTaskId));
            verified = true;
        } else if (jobChangeNotification.hasSnapshotEnd()) {
            break;
        }
    }
    assertThat(verified).isTrue();
}
Also used : TaskScenarioBuilder(com.netflix.titus.master.integration.v3.scenario.TaskScenarioBuilder) JobDescriptorGenerator.serviceJobDescriptors(com.netflix.titus.testkit.model.job.JobDescriptorGenerator.serviceJobDescriptors) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) ScenarioTemplates(com.netflix.titus.master.integration.v3.scenario.ScenarioTemplates) Function(java.util.function.Function) TaskStatus(com.netflix.titus.grpc.protogen.TaskStatus) SignedIpAddressAllocation(com.netflix.titus.api.jobmanager.model.job.vpc.SignedIpAddressAllocation) Map(java.util.Map) BatchJobExt(com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt) GrpcJobManagementModelConverters.toGrpcJobDescriptor(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobManagementModelConverters.toGrpcJobDescriptor) JobTestUtils.submitBadJob(com.netflix.titus.master.integration.v3.job.JobTestUtils.submitBadJob) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) Before(org.junit.Before) BaseIntegrationTest(com.netflix.titus.master.integration.BaseIntegrationTest) JobDescriptor(com.netflix.titus.api.jobmanager.model.job.JobDescriptor) Iterator(java.util.Iterator) ServiceJobExt(com.netflix.titus.api.jobmanager.model.job.ext.ServiceJobExt) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) JobFunctions(com.netflix.titus.api.jobmanager.model.job.JobFunctions) JobManagementServiceGrpc(com.netflix.titus.grpc.protogen.JobManagementServiceGrpc) IntegrationTest(com.netflix.titus.testkit.junit.category.IntegrationTest) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) Page(com.netflix.titus.grpc.protogen.Page) RuleChain(org.junit.rules.RuleChain) TitusStackResource(com.netflix.titus.testkit.junit.master.TitusStackResource) AbstractMap(java.util.AbstractMap) List(java.util.List) Rule(org.junit.Rule) JobDescriptorGenerator.batchJobDescriptors(com.netflix.titus.testkit.model.job.JobDescriptorGenerator.batchJobDescriptors) Ignore(org.junit.Ignore) JobIpAllocationGenerator(com.netflix.titus.testkit.model.job.JobIpAllocationGenerator) EmbeddedTitusCells.basicKubeCell(com.netflix.titus.testkit.embedded.cell.EmbeddedTitusCells.basicKubeCell) TaskAttributes(com.netflix.titus.api.jobmanager.TaskAttributes) JobsScenarioBuilder(com.netflix.titus.master.integration.v3.scenario.JobsScenarioBuilder) JobChangeNotification(com.netflix.titus.grpc.protogen.JobChangeNotification) JobId(com.netflix.titus.grpc.protogen.JobId) AbstractMap(java.util.AbstractMap) JobChangeNotification(com.netflix.titus.grpc.protogen.JobChangeNotification) ServiceJobExt(com.netflix.titus.api.jobmanager.model.job.ext.ServiceJobExt) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) Ignore(org.junit.Ignore) BaseIntegrationTest(com.netflix.titus.master.integration.BaseIntegrationTest) IntegrationTest(com.netflix.titus.testkit.junit.category.IntegrationTest) Test(org.junit.Test)

Example 5 with TaskQueryResult

use of com.netflix.titus.grpc.protogen.TaskQueryResult in project titus-control-plane by Netflix.

the class JobCriteriaQueryTest method testSearchByJobState.

@Test(timeout = 30_000)
public void testSearchByJobState() throws Exception {
    JobDescriptor<BatchJobExt> jobDescriptor = batchJobDescriptors().getValue().toBuilder().withApplicationName("testSearchByJobState").build();
    String acceptedJobId = jobsScenarioBuilder.scheduleAndReturnJob(jobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob())).getId();
    String killInitiatedJobId = jobsScenarioBuilder.scheduleAndReturnJob(jobDescriptor, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.launchJob()).killJob().expectJobUpdateEvent(job -> job.getStatus().getState() == JobState.KillInitiated, "Expected state: " + JobState.KillInitiated)).getId();
    String acceptedTaskId = jobsScenarioBuilder.takeJob(acceptedJobId).getTaskByIndex(0).getTask().getId();
    String killInitiatedTaskId = jobsScenarioBuilder.takeJob(killInitiatedJobId).getTaskByIndex(0).getTask().getId();
    // Indexes are recomputed after events are sent, so if we run findJobs/findTasks immediately, they may use stale index.
    Thread.sleep(10);
    JobQuery.Builder jobQueryBuilder = JobQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByJobState").setPage(PAGE);
    TaskQuery.Builder taskQueryBuilder = TaskQuery.newBuilder().putFilteringCriteria("applicationName", "testSearchByJobState").setPage(PAGE);
    // Jobs (Accepted)
    JobQueryResult acceptedJobQueryResult = client.findJobs(jobQueryBuilder.putFilteringCriteria("jobState", "Accepted").build());
    assertThat(acceptedJobQueryResult.getItemsList()).hasSize(1);
    Job acceptedJobQueryResultItem = acceptedJobQueryResult.getItems(0);
    assertThat(acceptedJobQueryResultItem.getId()).isEqualTo(acceptedJobId);
    // Jobs (KillInitiated)
    JobQueryResult killInitJobQueryResult = client.findJobs(jobQueryBuilder.putFilteringCriteria("jobState", "KillInitiated").setPage(PAGE).build());
    assertThat(killInitJobQueryResult.getItemsList()).hasSize(1);
    Job killInitJobQueryResultItem = killInitJobQueryResult.getItems(0);
    assertThat(killInitJobQueryResultItem.getId()).isEqualTo(killInitiatedJobId);
    // Tasks (Accepted)
    TaskQueryResult acceptedTaskQueryResult = client.findTasks(taskQueryBuilder.putFilteringCriteria("jobState", "Accepted").setPage(PAGE).build());
    assertThat(acceptedTaskQueryResult.getItemsList()).hasSize(1);
    assertThat(acceptedTaskQueryResult.getItems(0).getId()).isEqualTo(acceptedTaskId);
    // Tasks (KillInitiated)
    TaskQueryResult killInitTaskQueryResult = client.findTasks(taskQueryBuilder.putFilteringCriteria("jobState", "KillInitiated").setPage(PAGE).build());
    assertThat(killInitTaskQueryResult.getItemsList()).hasSize(1);
    assertThat(killInitTaskQueryResult.getItems(0).getId()).isEqualTo(killInitiatedTaskId);
}
Also used : TaskScenarioBuilder(com.netflix.titus.master.integration.v3.scenario.TaskScenarioBuilder) CollectionsExt(com.netflix.titus.common.util.CollectionsExt) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) ScenarioTemplates(com.netflix.titus.master.integration.v3.scenario.ScenarioTemplates) EmbeddedTitusMasters(com.netflix.titus.testkit.embedded.cell.master.EmbeddedTitusMasters) JobQueryResult(com.netflix.titus.grpc.protogen.JobQueryResult) JobGroupInfo(com.netflix.titus.api.jobmanager.model.job.JobGroupInfo) JobState(com.netflix.titus.api.jobmanager.model.job.JobState) EmbeddedKubeClusters(com.netflix.titus.testkit.embedded.kube.EmbeddedKubeClusters) ClassRule(org.junit.ClassRule) JobDescriptorGenerator.oneTaskServiceJobDescriptor(com.netflix.titus.testkit.model.job.JobDescriptorGenerator.oneTaskServiceJobDescriptor) BaseIntegrationTest(com.netflix.titus.master.integration.BaseIntegrationTest) ImmutableMap(com.google.common.collect.ImmutableMap) PlatformSidecar(com.netflix.titus.api.jobmanager.model.job.PlatformSidecar) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) Task(com.netflix.titus.grpc.protogen.Task) JobDescriptorGenerator.oneTaskBatchJobDescriptor(com.netflix.titus.testkit.model.job.JobDescriptorGenerator.oneTaskBatchJobDescriptor) Set(java.util.Set) JobFunctions(com.netflix.titus.api.jobmanager.model.job.JobFunctions) IntegrationTest(com.netflix.titus.testkit.junit.category.IntegrationTest) Category(org.junit.experimental.categories.Category) Collectors(java.util.stream.Collectors) Page(com.netflix.titus.grpc.protogen.Page) List(java.util.List) JobsScenarioBuilder(com.netflix.titus.master.integration.v3.scenario.JobsScenarioBuilder) CellAssertions.assertCellInfo(com.netflix.titus.master.integration.v3.job.CellAssertions.assertCellInfo) BeforeClass(org.junit.BeforeClass) Job(com.netflix.titus.grpc.protogen.Job) Image(com.netflix.titus.api.jobmanager.model.job.Image) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) JobQuery(com.netflix.titus.grpc.protogen.JobQuery) Owner(com.netflix.titus.api.jobmanager.model.job.Owner) TaskStatus(com.netflix.titus.grpc.protogen.TaskStatus) BatchJobExt(com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) JobDescriptor(com.netflix.titus.api.jobmanager.model.job.JobDescriptor) ServiceJobExt(com.netflix.titus.api.jobmanager.model.job.ext.ServiceJobExt) JobManagementServiceGrpc(com.netflix.titus.grpc.protogen.JobManagementServiceGrpc) Test(org.junit.Test) Pagination(com.netflix.titus.grpc.protogen.Pagination) JobScenarioBuilder(com.netflix.titus.master.integration.v3.scenario.JobScenarioBuilder) RuleChain(org.junit.rules.RuleChain) TitusStackResource(com.netflix.titus.testkit.junit.master.TitusStackResource) JobDescriptorGenerator.batchJobDescriptors(com.netflix.titus.testkit.model.job.JobDescriptorGenerator.batchJobDescriptors) EmbeddedTitusMaster(com.netflix.titus.testkit.embedded.cell.master.EmbeddedTitusMaster) EmbeddedTitusCell(com.netflix.titus.testkit.embedded.cell.EmbeddedTitusCell) Collections(java.util.Collections) TaskQuery(com.netflix.titus.grpc.protogen.TaskQuery) BatchJobExt(com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt) JobQuery(com.netflix.titus.grpc.protogen.JobQuery) Job(com.netflix.titus.grpc.protogen.Job) TaskQueryResult(com.netflix.titus.grpc.protogen.TaskQueryResult) JobQueryResult(com.netflix.titus.grpc.protogen.JobQueryResult) BaseIntegrationTest(com.netflix.titus.master.integration.BaseIntegrationTest) IntegrationTest(com.netflix.titus.testkit.junit.category.IntegrationTest) Test(org.junit.Test)

Aggregations

TaskQueryResult (com.netflix.titus.grpc.protogen.TaskQueryResult)29 Test (org.junit.Test)20 TaskQuery (com.netflix.titus.grpc.protogen.TaskQuery)13 BaseIntegrationTest (com.netflix.titus.master.integration.BaseIntegrationTest)12 IntegrationTest (com.netflix.titus.testkit.junit.category.IntegrationTest)12 Task (com.netflix.titus.grpc.protogen.Task)11 JobQueryResult (com.netflix.titus.grpc.protogen.JobQueryResult)9 Page (com.netflix.titus.grpc.protogen.Page)8 HashSet (java.util.HashSet)8 List (java.util.List)8 Job (com.netflix.titus.grpc.protogen.Job)7 ArrayList (java.util.ArrayList)7 Function (java.util.function.Function)6 JobQuery (com.netflix.titus.grpc.protogen.JobQuery)5 TaskStatus (com.netflix.titus.grpc.protogen.TaskStatus)5 JobDescriptor (com.netflix.titus.api.jobmanager.model.job.JobDescriptor)3 JobFunctions (com.netflix.titus.api.jobmanager.model.job.JobFunctions)3 BatchJobExt (com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt)3 ServiceJobExt (com.netflix.titus.api.jobmanager.model.job.ext.ServiceJobExt)3 Pair (com.netflix.titus.common.util.tuple.Pair)3