Examples with TaskRunnerWorkItem io.druid.indexing.overlord.TaskRunnerWorkItem used on opensource projects

Example 1 with TaskRunnerWorkItem

use of io.druid.indexing.overlord.TaskRunnerWorkItem in project druid by druid-io.

the class OverlordResource method securedTaskRunnerWorkItem.

private Collection<? extends TaskRunnerWorkItem> securedTaskRunnerWorkItem(Collection<? extends TaskRunnerWorkItem> collectionToFilter, HttpServletRequest req) {
    final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
    final AuthorizationInfo authorizationInfo = (AuthorizationInfo) req.getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
    return Collections2.filter(collectionToFilter, new Predicate<TaskRunnerWorkItem>() {

        @Override
        public boolean apply(TaskRunnerWorkItem input) {
            final String taskId = input.getTaskId();
            final Optional<Task> optionalTask = taskStorageQueryAdapter.getTask(taskId);
            if (!optionalTask.isPresent()) {
                throw new WebApplicationException(Response.serverError().entity(String.format("No task information found for task with id: [%s]", taskId)).build());
            }
            Resource resource = new Resource(optionalTask.get().getDataSource(), ResourceType.DATASOURCE);
            Action action = Action.READ;
            Pair<Resource, Action> key = new Pair<>(resource, action);
            if (resourceAccessMap.containsKey(key)) {
                return resourceAccessMap.get(key).isAllowed();
            } else {
                Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
                resourceAccessMap.put(key, access);
                return access.isAllowed();
            }
        }
    });
}

Example 2 with TaskRunnerWorkItem

use of io.druid.indexing.overlord.TaskRunnerWorkItem in project druid by druid-io.

the class SimpleWorkerResourceManagementStrategy method hasTaskPendingBeyondThreshold.

private boolean hasTaskPendingBeyondThreshold(Collection<? extends TaskRunnerWorkItem> pendingTasks) {
    synchronized (lock) {
        long now = System.currentTimeMillis();
        for (TaskRunnerWorkItem pendingTask : pendingTasks) {
            final Duration durationSinceInsertion = new Duration(pendingTask.getQueueInsertionTime().getMillis(), now);
            final Duration timeoutDuration = config.getPendingTaskTimeout().toStandardDuration();
            if (durationSinceInsertion.isEqual(timeoutDuration) || durationSinceInsertion.isLongerThan(timeoutDuration)) {
                return true;
            }
        }
        return false;
    }
}

Example 3 with TaskRunnerWorkItem

use of io.druid.indexing.overlord.TaskRunnerWorkItem in project druid by druid-io.

the class OverlordResource method getWaitingTasks.

@GET
@Path("/waitingTasks")
@Produces(MediaType.APPLICATION_JSON)
public Response getWaitingTasks(@Context final HttpServletRequest req) {
    return workItemsResponse(new Function<TaskRunner, Collection<? extends TaskRunnerWorkItem>>() {

        @Override
        public Collection<? extends TaskRunnerWorkItem> apply(TaskRunner taskRunner) {
            // A bit roundabout, but works as a way of figuring out what tasks haven't been handed
            // off to the runner yet:
            final List<Task> allActiveTasks = taskStorageQueryAdapter.getActiveTasks();
            final List<Task> activeTasks;
            if (authConfig.isEnabled()) {
                // This is an experimental feature, see - https://github.com/druid-io/druid/pull/2424
                final Map<Pair<Resource, Action>, Access> resourceAccessMap = new HashMap<>();
                final AuthorizationInfo authorizationInfo = (AuthorizationInfo) req.getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
                activeTasks = ImmutableList.copyOf(Iterables.filter(allActiveTasks, new Predicate<Task>() {

                    @Override
                    public boolean apply(Task input) {
                        Resource resource = new Resource(input.getDataSource(), ResourceType.DATASOURCE);
                        Action action = Action.READ;
                        Pair<Resource, Action> key = new Pair<>(resource, action);
                        if (resourceAccessMap.containsKey(key)) {
                            return resourceAccessMap.get(key).isAllowed();
                        } else {
                            Access access = authorizationInfo.isAuthorized(key.lhs, key.rhs);
                            resourceAccessMap.put(key, access);
                            return access.isAllowed();
                        }
                    }
                }));
            } else {
                activeTasks = allActiveTasks;
            }
            final Set<String> runnersKnownTasks = Sets.newHashSet(Iterables.transform(taskRunner.getKnownTasks(), new Function<TaskRunnerWorkItem, String>() {

                @Override
                public String apply(final TaskRunnerWorkItem workItem) {
                    return workItem.getTaskId();
                }
            }));
            final List<TaskRunnerWorkItem> waitingTasks = Lists.newArrayList();
            for (final Task task : activeTasks) {
                if (!runnersKnownTasks.contains(task.getId())) {
                    waitingTasks.add(// Would be nice to include the real created date, but the TaskStorage API doesn't yet allow it.
                    new TaskRunnerWorkItem(task.getId(), SettableFuture.<TaskStatus>create(), new DateTime(0), new DateTime(0)) {

                        @Override
                        public TaskLocation getLocation() {
                            return TaskLocation.unknown();
                        }
                    });
                }
            }
            return waitingTasks;
        }
    });
}