Examples with ObserveJobsQuery com.netflix.titus.grpc.protogen.ObserveJobsQuery used on opensource projects

Example 1 with ObserveJobsQuery

use of com.netflix.titus.grpc.protogen.ObserveJobsQuery in project titus-control-plane by Netflix.

the class ObserveJobsSubscription method tryInitialize.

private boolean tryInitialize() {
    ObserveJobsQuery query = getLastObserveJobsQueryEvent();
    if (query == null) {
        return false;
    }
    Stopwatch start = Stopwatch.createStarted();
    String trxId = UUID.randomUUID().toString();
    CallMetadata callMetadata = context.getCallMetadataResolver().resolve().orElse(CallMetadataConstants.UNDEFINED_CALL_METADATA);
    metrics.observeJobsStarted(trxId, callMetadata);
    JobQueryCriteria<TaskStatus.TaskState, JobDescriptor.JobSpecCase> criteria = toJobQueryCriteria(query);
    V3JobQueryCriteriaEvaluator jobsPredicate = new V3JobQueryCriteriaEvaluator(criteria, titusRuntime);
    V3TaskQueryCriteriaEvaluator tasksPredicate = new V3TaskQueryCriteriaEvaluator(criteria, titusRuntime);
    Observable<JobChangeNotification> eventStream = context.getJobOperations().observeJobs(jobsPredicate, tasksPredicate, true).filter(event -> withArchived || !event.isArchived()).observeOn(context.getObserveJobsScheduler()).subscribeOn(context.getObserveJobsScheduler(), false).map(event -> GrpcJobManagementModelConverters.toGrpcJobChangeNotification(event, context.getGrpcObjectsCache(), titusRuntime.getClock().wallTime())).compose(ObservableExt.head(() -> {
        List<JobChangeNotification> snapshot = createJobsSnapshot(jobsPredicate, tasksPredicate);
        snapshot.add(SNAPSHOT_END_MARKER);
        return snapshot;
    })).doOnError(e -> logger.error("Unexpected error in jobs event stream", e));
    AtomicBoolean closingProcessed = new AtomicBoolean();
    this.jobServiceSubscription = eventStream.doOnUnsubscribe(() -> {
        if (!closingProcessed.getAndSet(true)) {
            metrics.observeJobsUnsubscribed(trxId, start.elapsed(TimeUnit.MILLISECONDS));
        }
    }).subscribe(event -> {
        metrics.observeJobsEventEmitted(trxId);
        jobServiceEvents.add(event);
        drain();
    }, e -> {
        if (!closingProcessed.getAndSet(true)) {
            metrics.observeJobsError(trxId, start.elapsed(TimeUnit.MILLISECONDS), e);
        }
        jobServiceCompleted = true;
        jobServiceError = new StatusRuntimeException(Status.INTERNAL.withDescription("All jobs monitoring stream terminated with an error").withCause(e));
        drain();
    }, () -> {
        if (!closingProcessed.getAndSet(true)) {
            metrics.observeJobsCompleted(trxId, start.elapsed(TimeUnit.MILLISECONDS));
        }
        jobServiceCompleted = true;
        drain();
    });
    this.grpcStreamInitiated = true;
    return true;
}

Example 2 with ObserveJobsQuery

use of com.netflix.titus.grpc.protogen.ObserveJobsQuery in project titus-control-plane by Netflix.

the class JobObserveTest method observeSnapshotWithFilter.

@Test(timeout = LONG_TEST_TIMEOUT_MS)
public void observeSnapshotWithFilter() throws InterruptedException {
    startAll(batchJobDescriptors().getValue().toBuilder().withApplicationName("myApp").build(), batchJobDescriptors().getValue().toBuilder().withApplicationName("otherApp").build());
    String myAppJobId = jobsScenarioBuilder.takeJobId(0);
    // start the stream after tasks are already running
    ObserveJobsQuery query = ObserveJobsQuery.newBuilder().putFilteringCriteria("applicationName", "myApp").build();
    TestStreamObserver<JobChangeNotification> subscriber = subscribe(query);
    assertNextIsJobEvent(subscriber, job -> assertThat(job.getJobDescriptor().getApplicationName()).isEqualTo("myApp"));
    assertNextIsTaskEvent(subscriber, task -> assertThat(task.getJobId()).isEqualTo(myAppJobId));
    assertNextIsSnapshot(subscriber);
}

Example 3 with ObserveJobsQuery

use of com.netflix.titus.grpc.protogen.ObserveJobsQuery in project titus-control-plane by Netflix.

the class ObserveJobsCommand method executeWithFiltering.

private void executeWithFiltering(CommandContext context, Set<String> jobFields, Set<String> taskFields, boolean printEvents, boolean snapshotOnly) {
    JobManagementServiceBlockingStub stub = context.getJobManagementGrpcBlockingStub();
    Stopwatch stopwatch = Stopwatch.createStarted();
    ObserveJobsQuery query = ObserveJobsQuery.newBuilder().addAllJobFields(jobFields).addAllTaskFields(taskFields).build();
    Iterator<JobChangeNotification> eventIt = stub.observeJobs(query);
    while (eventIt.hasNext()) {
        JobChangeNotification next = eventIt.next();
        if (next.getNotificationCase() == JobChangeNotification.NotificationCase.SNAPSHOTEND) {
            logger.info("Emitted: snapshot marker in {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            if (snapshotOnly) {
                return;
            }
        } else if (next.getNotificationCase() == JobChangeNotification.NotificationCase.JOBUPDATE) {
            com.netflix.titus.grpc.protogen.Job job = next.getJobUpdate().getJob();
            if (printEvents) {
                logger.info("Emitted job update: jobId={}({}), jobState={}, version={}", job.getId(), next.getJobUpdate().getArchived() ? "archived" : job.getStatus().getState(), job.getStatus(), job.getVersion());
            }
        } else if (next.getNotificationCase() == JobChangeNotification.NotificationCase.TASKUPDATE) {
            com.netflix.titus.grpc.protogen.Task task = next.getTaskUpdate().getTask();
            if (printEvents) {
                logger.info("Emitted task update: jobId={}({}), taskId={}, taskState={}, version={}", task.getJobId(), next.getTaskUpdate().getArchived() ? "archived" : task.getStatus().getState(), task.getId(), task.getStatus(), task.getVersion());
            }
        } else {
            logger.info("Unrecognized event type: {}", next);
        }
    }
}

Example 4 with ObserveJobsQuery

use of com.netflix.titus.grpc.protogen.ObserveJobsQuery in project titus-control-plane by Netflix.

the class ObserveJobsSubscription method getLastObserveJobsQueryEvent.

private ObserveJobsQuery getLastObserveJobsQueryEvent() {
    ObserveJobsQuery jobsQuery = null;
    Pair<Long, ObserveJobsWithKeepAliveRequest> eventPair;
    while ((eventPair = grpcClientEvents.poll()) != null) {
        ObserveJobsWithKeepAliveRequest event = eventPair.getRight();
        if (event.getKindCase() == ObserveJobsWithKeepAliveRequest.KindCase.QUERY) {
            jobsQuery = event.getQuery();
        }
    }
    return jobsQuery;
}

Example 5 with ObserveJobsQuery

use of com.netflix.titus.grpc.protogen.ObserveJobsQuery in project titus-control-plane by Netflix.

the class LocalCacheQueryProcessorTest method testObserveJobsWithFieldsFiltering.

@Test
public void testObserveJobsWithFieldsFiltering() throws InterruptedException {
    Pair<Job<?>, List<Task>> jobAndTasks = addToJobDataReplicator(newJobAndTasks("job1", 1));
    Job<?> job1 = jobAndTasks.getLeft();
    Task task1 = jobAndTasks.getRight().get(0);
    ExtTestSubscriber<JobChangeNotification> subscriber = new ExtTestSubscriber<>();
    ObserveJobsQuery query = ObserveJobsQuery.newBuilder().addJobFields("status").addTaskFields("taskContext").build();
    processor.observeJobs(query).subscribe(subscriber);
    // Snapshot
    emitEvent(Pair.of(jobDataReplicator.getCurrent(), JobManagerEvent.snapshotMarker()));
    expectJobUpdateEvent(subscriber, j -> {
        assertThat(j.getId()).isEqualTo(job1.getId());
        assertThat(j.getStatus().getReasonMessage()).isEqualTo("<not_given>");
        assertThat(j.getJobDescriptor()).isEqualTo(JobDescriptor.getDefaultInstance());
    });
    expectTaskUpdateEvent(subscriber, t -> {
        assertThat(t.getId()).isEqualTo(task1.getId());
        assertThat(t.getStatus()).isEqualTo(TaskStatus.getDefaultInstance());
        assertThat(t.getTaskContextMap()).containsAllEntriesOf(task1.getTaskContext());
    });
    expectSnapshot(subscriber);
    // Updates after snapshot marker
    Pair<Job<?>, List<Task>> jobAndTasks2 = addToJobDataReplicator(newJobAndTasks("job2", 1));
    Job<?> job2 = jobAndTasks2.getLeft();
    Task task2 = jobAndTasks2.getRight().get(0);
    JobUpdateEvent jobUpdateEvent = JobUpdateEvent.newJob(job2, JUNIT_CALL_METADATA);
    TaskUpdateEvent taskUpdateEvent = TaskUpdateEvent.newTask(job2, task2, JUNIT_CALL_METADATA);
    emitEvent(Pair.of(jobDataReplicator.getCurrent(), jobUpdateEvent));
    emitEvent(Pair.of(jobDataReplicator.getCurrent(), taskUpdateEvent));
    expectJobUpdateEvent(subscriber, j -> {
        assertThat(j.getId()).isEqualTo(job2.getId());
        assertThat(j.getStatus().getReasonMessage()).isEqualTo("<not_given>");
        assertThat(j.getJobDescriptor()).isEqualTo(JobDescriptor.getDefaultInstance());
    });
    expectTaskUpdateEvent(subscriber, t -> {
        assertThat(t.getId()).isEqualTo(task2.getId());
        assertThat(t.getStatus()).isEqualTo(TaskStatus.getDefaultInstance());
        assertThat(t.getTaskContextMap()).containsAllEntriesOf(task2.getTaskContext());
    });
}