Search in sources :

Example 1 with SNAPSHOT_END_MARKER

use of com.netflix.titus.master.jobmanager.endpoint.v3.grpc.ObserveJobsContext.SNAPSHOT_END_MARKER in project titus-control-plane by Netflix.

the class ObserveJobsSubscription method tryInitialize.

private boolean tryInitialize() {
    ObserveJobsQuery query = getLastObserveJobsQueryEvent();
    if (query == null) {
        return false;
    }
    Stopwatch start = Stopwatch.createStarted();
    String trxId = UUID.randomUUID().toString();
    CallMetadata callMetadata = context.getCallMetadataResolver().resolve().orElse(CallMetadataConstants.UNDEFINED_CALL_METADATA);
    metrics.observeJobsStarted(trxId, callMetadata);
    JobQueryCriteria<TaskStatus.TaskState, JobDescriptor.JobSpecCase> criteria = toJobQueryCriteria(query);
    V3JobQueryCriteriaEvaluator jobsPredicate = new V3JobQueryCriteriaEvaluator(criteria, titusRuntime);
    V3TaskQueryCriteriaEvaluator tasksPredicate = new V3TaskQueryCriteriaEvaluator(criteria, titusRuntime);
    Observable<JobChangeNotification> eventStream = context.getJobOperations().observeJobs(jobsPredicate, tasksPredicate, true).filter(event -> withArchived || !event.isArchived()).observeOn(context.getObserveJobsScheduler()).subscribeOn(context.getObserveJobsScheduler(), false).map(event -> GrpcJobManagementModelConverters.toGrpcJobChangeNotification(event, context.getGrpcObjectsCache(), titusRuntime.getClock().wallTime())).compose(ObservableExt.head(() -> {
        List<JobChangeNotification> snapshot = createJobsSnapshot(jobsPredicate, tasksPredicate);
        snapshot.add(SNAPSHOT_END_MARKER);
        return snapshot;
    })).doOnError(e -> logger.error("Unexpected error in jobs event stream", e));
    AtomicBoolean closingProcessed = new AtomicBoolean();
    this.jobServiceSubscription = eventStream.doOnUnsubscribe(() -> {
        if (!closingProcessed.getAndSet(true)) {
            metrics.observeJobsUnsubscribed(trxId, start.elapsed(TimeUnit.MILLISECONDS));
        }
    }).subscribe(event -> {
        metrics.observeJobsEventEmitted(trxId);
        jobServiceEvents.add(event);
        drain();
    }, e -> {
        if (!closingProcessed.getAndSet(true)) {
            metrics.observeJobsError(trxId, start.elapsed(TimeUnit.MILLISECONDS), e);
        }
        jobServiceCompleted = true;
        jobServiceError = new StatusRuntimeException(Status.INTERNAL.withDescription("All jobs monitoring stream terminated with an error").withCause(e));
        drain();
    }, () -> {
        if (!closingProcessed.getAndSet(true)) {
            metrics.observeJobsCompleted(trxId, start.elapsed(TimeUnit.MILLISECONDS));
        }
        jobServiceCompleted = true;
        drain();
    });
    this.grpcStreamInitiated = true;
    return true;
}
Also used : KeepAliveResponse(com.netflix.titus.grpc.protogen.KeepAliveResponse) Stopwatch(com.google.common.base.Stopwatch) ObserveJobsQuery(com.netflix.titus.grpc.protogen.ObserveJobsQuery) Task(com.netflix.titus.api.jobmanager.model.job.Task) CallMetadataConstants(com.netflix.titus.api.model.callmetadata.CallMetadataConstants) LoggerFactory(org.slf4j.LoggerFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayList(java.util.ArrayList) Observable(rx.Observable) StreamObserver(io.grpc.stub.StreamObserver) Pair(com.netflix.titus.common.util.tuple.Pair) TaskStatus(com.netflix.titus.grpc.protogen.TaskStatus) ExceptionExt(com.netflix.titus.common.util.ExceptionExt) Status(io.grpc.Status) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) ServerCallStreamObserver(io.grpc.stub.ServerCallStreamObserver) JobDescriptor(com.netflix.titus.grpc.protogen.JobDescriptor) V3TaskQueryCriteriaEvaluator(com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator) Job(com.netflix.titus.api.jobmanager.model.job.Job) KeepAliveRequest(com.netflix.titus.grpc.protogen.KeepAliveRequest) Logger(org.slf4j.Logger) Predicate(java.util.function.Predicate) SNAPSHOT_END_MARKER(com.netflix.titus.master.jobmanager.endpoint.v3.grpc.ObserveJobsContext.SNAPSHOT_END_MARKER) BlockingQueue(java.util.concurrent.BlockingQueue) UUID(java.util.UUID) GrpcJobManagementModelConverters(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobManagementModelConverters) TimeUnit(java.util.concurrent.TimeUnit) StatusRuntimeException(io.grpc.StatusRuntimeException) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) JobQueryCriteria(com.netflix.titus.runtime.endpoint.JobQueryCriteria) LinkedBlockingDeque(java.util.concurrent.LinkedBlockingDeque) V3JobQueryCriteriaEvaluator(com.netflix.titus.runtime.endpoint.v3.grpc.query.V3JobQueryCriteriaEvaluator) GrpcJobQueryModelConverters.toJobQueryCriteria(com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobQueryModelConverters.toJobQueryCriteria) ObserveJobsWithKeepAliveRequest(com.netflix.titus.grpc.protogen.ObserveJobsWithKeepAliveRequest) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TitusRuntime(com.netflix.titus.common.runtime.TitusRuntime) ObservableExt(com.netflix.titus.common.util.rx.ObservableExt) Subscription(rx.Subscription) JobChangeNotification(com.netflix.titus.grpc.protogen.JobChangeNotification) V3TaskQueryCriteriaEvaluator(com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator) CallMetadata(com.netflix.titus.api.model.callmetadata.CallMetadata) Stopwatch(com.google.common.base.Stopwatch) V3JobQueryCriteriaEvaluator(com.netflix.titus.runtime.endpoint.v3.grpc.query.V3JobQueryCriteriaEvaluator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) JobChangeNotification(com.netflix.titus.grpc.protogen.JobChangeNotification) StatusRuntimeException(io.grpc.StatusRuntimeException) ObserveJobsQuery(com.netflix.titus.grpc.protogen.ObserveJobsQuery)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Stopwatch (com.google.common.base.Stopwatch)1 Job (com.netflix.titus.api.jobmanager.model.job.Job)1 Task (com.netflix.titus.api.jobmanager.model.job.Task)1 CallMetadata (com.netflix.titus.api.model.callmetadata.CallMetadata)1 CallMetadataConstants (com.netflix.titus.api.model.callmetadata.CallMetadataConstants)1 TitusRuntime (com.netflix.titus.common.runtime.TitusRuntime)1 ExceptionExt (com.netflix.titus.common.util.ExceptionExt)1 ObservableExt (com.netflix.titus.common.util.rx.ObservableExt)1 Pair (com.netflix.titus.common.util.tuple.Pair)1 JobChangeNotification (com.netflix.titus.grpc.protogen.JobChangeNotification)1 JobDescriptor (com.netflix.titus.grpc.protogen.JobDescriptor)1 KeepAliveRequest (com.netflix.titus.grpc.protogen.KeepAliveRequest)1 KeepAliveResponse (com.netflix.titus.grpc.protogen.KeepAliveResponse)1 ObserveJobsQuery (com.netflix.titus.grpc.protogen.ObserveJobsQuery)1 ObserveJobsWithKeepAliveRequest (com.netflix.titus.grpc.protogen.ObserveJobsWithKeepAliveRequest)1 TaskStatus (com.netflix.titus.grpc.protogen.TaskStatus)1 SNAPSHOT_END_MARKER (com.netflix.titus.master.jobmanager.endpoint.v3.grpc.ObserveJobsContext.SNAPSHOT_END_MARKER)1 JobQueryCriteria (com.netflix.titus.runtime.endpoint.JobQueryCriteria)1 GrpcJobManagementModelConverters (com.netflix.titus.runtime.endpoint.v3.grpc.GrpcJobManagementModelConverters)1