use of com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator in project titus-control-plane by Netflix.
the class DefaultJobManagementServiceGrpc method findTasks.
@Override
public void findTasks(TaskQuery taskQuery, StreamObserver<TaskQueryResult> responseObserver) {
if (!checkPageIsValid(taskQuery.getPage(), responseObserver)) {
return;
}
try {
// We need to find all tasks to get the total number of them.
List<com.netflix.titus.api.jobmanager.model.job.Task> allFilteredTasks = new ArrayList<>();
for (Pair<com.netflix.titus.api.jobmanager.model.job.Job<?>, com.netflix.titus.api.jobmanager.model.job.Task> jobTaskPair : jobOperations.findTasks(new V3TaskQueryCriteriaEvaluator(toJobQueryCriteria(taskQuery), titusRuntime), 0, Integer.MAX_VALUE / 2)) {
com.netflix.titus.api.jobmanager.model.job.Task right = jobTaskPair.getRight();
allFilteredTasks.add(right);
}
Pair<List<com.netflix.titus.api.jobmanager.model.job.Task>, Pagination> queryResult = PaginationUtil.takePageWithCursorAndKeyExtractor(toPage(taskQuery.getPage()), allFilteredTasks, JobComparators::createTaskKeyOf, JobManagerCursors::coreTaskIndexOf, JobManagerCursors::newTaskCoreCursorFrom);
List<Task> grpcTasks = new ArrayList<>();
for (com.netflix.titus.api.jobmanager.model.job.Task task : queryResult.getLeft()) {
Task toGrpcTask = grpcObjectsCache.getTask(task);
grpcTasks.add(toGrpcTask);
}
TaskQueryResult grpcQueryResult;
if (taskQuery.getFieldsList().isEmpty()) {
grpcQueryResult = toTaskQueryResult(grpcTasks, queryResult.getRight());
} else {
Set<String> fields = new HashSet<>(taskQuery.getFieldsList());
fields.addAll(TASK_MINIMUM_FIELD_SET);
List<Task> filtered = new ArrayList<>();
for (Task t : grpcTasks) {
filtered.add(ProtobufExt.copy(t, fields));
}
grpcQueryResult = toTaskQueryResult(filtered, queryResult.getRight());
}
responseObserver.onNext(grpcQueryResult);
responseObserver.onCompleted();
} catch (Exception e) {
safeOnError(logger, e, responseObserver);
}
}
use of com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator in project titus-control-plane by Netflix.
the class ObserveJobsSubscription method tryInitialize.
private boolean tryInitialize() {
ObserveJobsQuery query = getLastObserveJobsQueryEvent();
if (query == null) {
return false;
}
Stopwatch start = Stopwatch.createStarted();
String trxId = UUID.randomUUID().toString();
CallMetadata callMetadata = context.getCallMetadataResolver().resolve().orElse(CallMetadataConstants.UNDEFINED_CALL_METADATA);
metrics.observeJobsStarted(trxId, callMetadata);
JobQueryCriteria<TaskStatus.TaskState, JobDescriptor.JobSpecCase> criteria = toJobQueryCriteria(query);
V3JobQueryCriteriaEvaluator jobsPredicate = new V3JobQueryCriteriaEvaluator(criteria, titusRuntime);
V3TaskQueryCriteriaEvaluator tasksPredicate = new V3TaskQueryCriteriaEvaluator(criteria, titusRuntime);
Observable<JobChangeNotification> eventStream = context.getJobOperations().observeJobs(jobsPredicate, tasksPredicate, true).filter(event -> withArchived || !event.isArchived()).observeOn(context.getObserveJobsScheduler()).subscribeOn(context.getObserveJobsScheduler(), false).map(event -> GrpcJobManagementModelConverters.toGrpcJobChangeNotification(event, context.getGrpcObjectsCache(), titusRuntime.getClock().wallTime())).compose(ObservableExt.head(() -> {
List<JobChangeNotification> snapshot = createJobsSnapshot(jobsPredicate, tasksPredicate);
snapshot.add(SNAPSHOT_END_MARKER);
return snapshot;
})).doOnError(e -> logger.error("Unexpected error in jobs event stream", e));
AtomicBoolean closingProcessed = new AtomicBoolean();
this.jobServiceSubscription = eventStream.doOnUnsubscribe(() -> {
if (!closingProcessed.getAndSet(true)) {
metrics.observeJobsUnsubscribed(trxId, start.elapsed(TimeUnit.MILLISECONDS));
}
}).subscribe(event -> {
metrics.observeJobsEventEmitted(trxId);
jobServiceEvents.add(event);
drain();
}, e -> {
if (!closingProcessed.getAndSet(true)) {
metrics.observeJobsError(trxId, start.elapsed(TimeUnit.MILLISECONDS), e);
}
jobServiceCompleted = true;
jobServiceError = new StatusRuntimeException(Status.INTERNAL.withDescription("All jobs monitoring stream terminated with an error").withCause(e));
drain();
}, () -> {
if (!closingProcessed.getAndSet(true)) {
metrics.observeJobsCompleted(trxId, start.elapsed(TimeUnit.MILLISECONDS));
}
jobServiceCompleted = true;
drain();
});
this.grpcStreamInitiated = true;
return true;
}
use of com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator in project titus-control-plane by Netflix.
the class LocalCacheQueryProcessor method observeJobs.
public Observable<JobChangeNotification> observeJobs(ObserveJobsQuery query) {
JobQueryCriteria<TaskStatus.TaskState, JobDescriptor.JobSpecCase> criteria = toJobQueryCriteria(query);
V3JobQueryCriteriaEvaluator jobsPredicate = new V3JobQueryCriteriaEvaluator(criteria, titusRuntime);
V3TaskQueryCriteriaEvaluator tasksPredicate = new V3TaskQueryCriteriaEvaluator(criteria, titusRuntime);
Set<String> jobFields = newFieldsFilter(query.getJobFieldsList(), JOB_MINIMUM_FIELD_SET);
Set<String> taskFields = newFieldsFilter(query.getTaskFieldsList(), TASK_MINIMUM_FIELD_SET);
Flux<JobChangeNotification> eventStream = Flux.defer(() -> {
AtomicBoolean first = new AtomicBoolean(true);
return jobDataReplicator.events().subscribeOn(scheduler).publishOn(scheduler).flatMap(event -> {
JobManagerEvent<?> jobManagerEvent = event.getRight();
long now = titusRuntime.getClock().wallTime();
JobSnapshot snapshot = event.getLeft();
Optional<JobChangeNotification> grpcEvent = toObserveJobsEvent(snapshot, jobManagerEvent, now, jobsPredicate, tasksPredicate, jobFields, taskFields);
// On first event emit full snapshot first
if (first.getAndSet(false)) {
List<JobChangeNotification> snapshotEvents = buildSnapshot(snapshot, now, jobsPredicate, tasksPredicate, jobFields, taskFields);
grpcEvent.ifPresent(snapshotEvents::add);
return Flux.fromIterable(snapshotEvents);
}
// subscribe again. Snapshot marker indicates that the underlying GRPC stream was disconnected.
if (jobManagerEvent == JobManagerEvent.snapshotMarker()) {
return Mono.error(new StatusRuntimeException(Status.ABORTED.augmentDescription("Downstream event stream reconnected.")));
}
// to filter them out here.
if (jobManagerEvent instanceof JobKeepAliveEvent) {
// Check if staleness is not too high.
if (jobDataReplicator.getStalenessMs() > configuration.getObserveJobsStalenessDisconnectMs()) {
rejectedByStalenessTooHighMetric.increment();
return Mono.error(new StatusRuntimeException(Status.ABORTED.augmentDescription("Data staleness in the event stream is too high. Most likely caused by connectivity issue to the downstream server.")));
}
return Mono.empty();
}
return grpcEvent.map(Flux::just).orElseGet(Flux::empty);
});
});
return ReactorExt.toObservable(eventStream);
}
use of com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator in project titus-control-plane by Netflix.
the class LocalCacheQueryProcessor method findMatchingTasks.
private List<com.netflix.titus.api.jobmanager.model.job.Task> findMatchingTasks(JobQueryCriteria<TaskStatus.TaskState, JobDescriptor.JobSpecCase> queryCriteria) {
JobSnapshot jobSnapshot = jobDataReplicator.getCurrent();
Map<String, Job<?>> jobsById = jobSnapshot.getJobMap();
V3TaskQueryCriteriaEvaluator queryFilter = new V3TaskQueryCriteriaEvaluator(queryCriteria, titusRuntime);
List<com.netflix.titus.api.jobmanager.model.job.Task> matchingTasks = new ArrayList<>();
jobsById.forEach((jobId, job) -> {
Map<String, com.netflix.titus.api.jobmanager.model.job.Task> tasks = jobSnapshot.getTasks(jobId);
if (!CollectionsExt.isNullOrEmpty(tasks)) {
tasks.forEach((taskId, task) -> {
Pair<Job<?>, com.netflix.titus.api.jobmanager.model.job.Task> jobTaskPair = Pair.of(job, task);
if (queryFilter.test(jobTaskPair)) {
matchingTasks.add(task);
}
});
}
});
return matchingTasks;
}
use of com.netflix.titus.runtime.endpoint.v3.grpc.query.V3TaskQueryCriteriaEvaluator in project titus-control-plane by Netflix.
the class NeedsMigrationQueryHandler method findMatchingTasks.
private List<com.netflix.titus.grpc.protogen.Task> findMatchingTasks(JobQueryCriteria<TaskStatus.TaskState, JobDescriptor.JobSpecCase> queryCriteria) {
JobSnapshot jobSnapshot = jobDataReplicator.getCurrent();
Map<String, Job<?>> jobMap = jobSnapshot.getJobMap();
Map<String, TaskRelocationPlan> relocationPlans = relocationDataReplicator.getCurrent().getPlans();
V3TaskQueryCriteriaEvaluator queryFilter = new V3TaskQueryCriteriaEvaluator(queryCriteria, titusRuntime);
V3TaskQueryCriteriaEvaluator queryFilterWithoutNeedsMigration = new V3TaskQueryCriteriaEvaluator(filterOutNeedsMigration(queryCriteria), titusRuntime);
List<com.netflix.titus.grpc.protogen.Task> matchingTasks = new ArrayList<>();
jobMap.forEach((jobId, job) -> {
Map<String, Task> tasks = jobSnapshot.getTasks(jobId);
if (!CollectionsExt.isNullOrEmpty(tasks)) {
tasks.forEach((taskId, task) -> {
TaskRelocationPlan plan = relocationPlans.get(task.getId());
Pair<Job<?>, Task> jobTaskPair = Pair.of(job, task);
if (plan != null) {
if (queryFilterWithoutNeedsMigration.test(jobTaskPair)) {
matchingTasks.add(newTaskWithRelocationPlan(GrpcJobManagementModelConverters.toGrpcTask(task, logStorageInfo), plan));
}
} else {
if (queryFilter.test(jobTaskPair)) {
matchingTasks.add(GrpcJobManagementModelConverters.toGrpcTask(task, logStorageInfo));
}
}
});
}
});
return matchingTasks;
}
Aggregations